Kaggle competition: Google QUEST Q&A Labeling
def match(h1, h2):
for p1 in h2ps[h1]:
for p2 in h2ps[h2]:
i1 = pil_image.open(expand_path(p1))
i2 = pil_image.open(expand_path(p2))
if i1.mode != i2.mode or i1.size != i2.size: return False
a1 = np.array(i1)
 a1 = a1 - a1.mean()
a1 = a1 / sqrt(( a1 ** 2 ).mean())
 a2 = np.array(i2)
 a2 = a2 - a2.mean()
a2 = a2 / sqrt(( a2 ** 2 ).mean())
 a =(( a1 - a2)** 2 ).mean()
if a > 0.1: return False
return True
if isfile(P2H):
print("P2H exists.")
 with open(P2H, 'rb')as f:
p2h = pickle.load(f)
 else:
p2h = {}
for p in tqdm(join):
img = pil_image.open(expand_path(p))
h = phash(img)
 p2h[p] = h
h2ps = {}
for p, h in p2h.items() :
if h not in h2ps: h2ps[h] = []
if p not in h2ps[h]: h2ps[h].append(p)
 hs = list(h2ps.keys())
 h2h = {}
for i, h1 in enumerate(tqdm(hs)) :
for h2 in hs[:i]:
if h1 - h2 <= 6 and match(h1, h2):
s1 = str(h1)
 s2 = str(h2)
 if s1 < s2: s1, s2 = s2, s1
h2h[s1] = s2
for p, h in p2h.items() :
h = str(h)
 if h in h2h: h = h2h[h]
p2h[p] = h
h2ps = {}
for p, h in p2h.items() :
if h not in h2ps: h2ps[h] = []
if p not in h2ps[h]: h2ps[h].append(p )<categorify>
train = pd.read_csv('/kaggle/input/google-quest-challenge/train.csv')
 test = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv')
 sample_submission = pd.read_csv('/kaggle/input/google-quest-challenge/sample_submission.csv' )
def prefer(ps):
if len(ps)== 1: return ps[0]
best_p = ps[0]
best_s = p2size[best_p]
for i in range(1, len(ps)) :
p = ps[i]
s = p2size[p]
if s[0] * s[1] > best_s[0] * best_s[1]:
best_p = p
best_s = s
return best_p
h2p = {}
for h, ps in h2ps.items() :
h2p[h] = prefer(ps)
 len(h2p), list(h2p.items())[:5]<set_options>
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-uncased-huggingface-transformer/'
tokenizer = BertTokenizer.from_pretrained(BERT_PATH+'bert-base-uncased-vocab.txt')
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )
p2bb = pd.read_csv(BB_DF ).set_index("Image")
 old_stderr = sys.stderr
sys.stderr = open('/dev/null' if platform.system() != 'Windows' else 'nul', 'w')
 sys.stderr = old_stderr
img_shape =(384, 384, 1)
 anisotropy = 2.15
crop_margin = 0.05<normalization>
!pip install.. /input/sacremoses > /dev/null
sys.path.insert(0, ".. /input/transformers/" )
def build_transform(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):

 rotation = np.deg2rad(rotation)
 shear = np.deg2rad(shear)
 rotation_matrix = np.array(
 [[np.cos(rotation), np.sin(rotation), 0], [-np.sin(rotation), np.cos(rotation), 0], [0, 0, 1]])
 shift_matrix = np.array([[1, 0, height_shift], [0, 1, width_shift], [0, 0, 1]])
 shear_matrix = np.array([[1, np.sin(shear), 0], [0, np.cos(shear), 0], [0, 0, 1]])
 zoom_matrix = np.array([[1.0 / height_zoom, 0, 0], [0, 1.0 / width_zoom, 0], [0, 0, 1]])
 shift_matrix = np.array([[1, 0, -height_shift], [0, 1, -width_shift], [0, 0, 1]])
 return np.dot(np.dot(rotation_matrix, shear_matrix), np.dot(zoom_matrix, shift_matrix))<prepare_x_and_y>
target_cols = ['question_asker_intent_understanding',
'question_body_critical', 'question_conversational',
'question_expect_short_answer', 'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others', 'question_interestingness_self',
'question_multi_intent', 'question_not_really_a_question',
'question_opinion_seeking', 'question_type_choice',
'question_type_compare', 'question_type_consequence',
'question_type_definition', 'question_type_entity',
'question_type_instructions', 'question_type_procedure',
'question_type_reason_explanation', 'question_type_spelling',
'question_well_written', 'answer_helpful',
'answer_level_of_information', 'answer_plausible', 'answer_relevance',
'answer_satisfaction', 'answer_type_instructions',
'answer_type_procedure', 'answer_type_reason_explanation',
'answer_well_written']
def read_cropped_image(p, augment):

 if p in h2p:
p = h2p[p]
size_x, size_y = p2size[p]
row = p2bb.loc[p]
x0, y0, x1, y1 = row['x0'], row['y0'], row['x1'], row['y1']
dx = x1 - x0
dy = y1 - y0
x0 -= dx * crop_margin
x1 += dx * crop_margin + 1
y0 -= dy * crop_margin
y1 += dy * crop_margin + 1
if x0 < 0:
x0 = 0
if x1 > size_x:
x1 = size_x
if y0 < 0:
y0 = 0
if y1 > size_y:
y1 = size_y
dx = x1 - x0
dy = y1 - y0
if dx > dy * anisotropy:
dy = 0.5 *(dx / anisotropy - dy)
 y0 -= dy
y1 += dy
else:
dx = 0.5 *(dy * anisotropy - dx)
 x0 -= dx
x1 += dx
trans = np.array([[1, 0, -0.5 * img_shape[0]], [0, 1, -0.5 * img_shape[1]], [0, 0, 1]])
 trans = np.dot(np.array([[(y1 - y0)/ img_shape[0], 0, 0], [0,(x1 - x0)/ img_shape[1], 0], [0, 0, 1]]), trans)
 if augment:
trans = np.dot(build_transform(
 random.uniform(-5, 5),
random.uniform(-5, 5),
random.uniform(0.8, 1.0),
random.uniform(0.8, 1.0),
random.uniform(-0.05 *(y1 - y0), 0.05 *(y1 - y0)) ,
random.uniform(-0.05 *(x1 - x0), 0.05 *(x1 - x0)) 
), trans)
 trans = np.dot(np.array([[1, 0, 0.5 *(y1 + y0)], [0, 1, 0.5 *(x1 + x0)], [0, 0, 1]]), trans)
 img = read_raw_image(p ).convert('L')
 img = img_to_array(img)
 matrix = trans[:2, :2]
offset = trans[:2, 2]
img = img.reshape(img.shape[:-1])
 img = affine_transform(img, matrix, offset, output_shape=img_shape[:-1], order=1, mode='constant',
cval=np.average(img))
img = img.reshape(img_shape)
 img -= np.mean(img, keepdims=True)
 img /= np.std(img, keepdims=True)+ K.epsilon()
return img
def read_for_training(p):

 return read_cropped_image(p, True)
 def read_for_validation(p):

 return read_cropped_image(p, False)
 p = list(tagged.keys())[312]
print(p )<import_modules>
train.isna().sum()
<categorify>
train_category = train['category'].value_counts()
test_category = test['category'].value_counts()
def subblock(x, filter, **kwargs):
x = BatchNormalization()(x)
 y = x
y = Conv2D(filter,(1, 1), activation='relu', **kwargs )(y)
 y = BatchNormalization()(y)
 y = Conv2D(filter,(3, 3), activation='relu', **kwargs )(y)
 y = BatchNormalization()(y)
 y = Conv2D(K.int_shape(x)[-1],(1, 1), **kwargs )(y)
 y = Add()([x, y])
 y = Activation('relu' )(y)
 return y
def build_model(lr, l2, activation='sigmoid'):
regul = regularizers.l2(l2)
 optim = Adam(lr=lr)
 kwargs = {'padding': 'same', 'kernel_regularizer': regul}
inp = Input(shape=img_shape)
 x = Conv2D(64,(9, 9), strides=2, activation='relu', **kwargs )(inp)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 for _ in range(2):
x = BatchNormalization()(x)
 x = Conv2D(64,(3, 3), activation='relu', **kwargs )(x)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(128,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 64, **kwargs)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(256,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 64, **kwargs)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(384,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 96, **kwargs)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(512,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 128, **kwargs)
 x = GlobalMaxPooling2D()(x)
 branch_model = Model(inp, x)
 mid = 32
xa_inp = Input(shape=branch_model.output_shape[1:])
 xb_inp = Input(shape=branch_model.output_shape[1:])
 x1 = Lambda(lambda x: x[0] * x[1] )([xa_inp, xb_inp])
 x2 = Lambda(lambda x: x[0] + x[1] )([xa_inp, xb_inp])
 x3 = Lambda(lambda x: K.abs(x[0] - x[1]))([xa_inp, xb_inp])
 x4 = Lambda(lambda x: K.square(x))(x3)
 x = Concatenate()([x1, x2, x3, x4])
 x = Reshape(( 4, branch_model.output_shape[1], 1), name='reshape1' )(x)
 x = Conv2D(mid,(4, 1), activation='relu', padding='valid' )(x)
 x = Reshape(( branch_model.output_shape[1], mid, 1))(x)
 x = Conv2D(1,(1, mid), activation='linear', padding='valid' )(x)
 x = Flatten(name='flatten' )(x)
 x = Dense(1, use_bias=True, activation=activation, name='weighted-average' )(x)
 head_model = Model([xa_inp, xb_inp], x, name='head')
 img_a = Input(shape=img_shape)
 img_b = Input(shape=img_shape)
 xa = branch_model(img_a)
 xb = branch_model(img_b)
 x = head_model([xa, xb])
 model = Model([img_a, img_b], x)
 model.compile(optim, loss='binary_crossentropy', metrics=['binary_crossentropy', 'acc'])
 return model, branch_model, head_model
model, branch_model, head_model = build_model(64e-5, 0 )<feature_engineering>
def _convert_to_transformer_inputs(title, question, answer, tokenizer, max_sequence_length):

 def return_id(str1, str2, truncation_strategy, length):
inputs = tokenizer.encode_plus(str1, str2,
add_special_tokens=True,
max_length=length,
truncation_strategy=truncation_strategy)
 input_ids = inputs["input_ids"]
input_masks = [1] * len(input_ids)
 input_segments = inputs["token_type_ids"]
padding_length = length - len(input_ids)
 padding_id = tokenizer.pad_token_id
input_ids = input_ids +([padding_id] * padding_length)
 input_masks = input_masks +([0] * padding_length)
 input_segments = input_segments +([0] * padding_length)
 return [input_ids, input_masks, input_segments]
input_ids_q, input_masks_q, input_segments_q = return_id(
 title + ' ' + question, None, 'longest_first', max_sequence_length)
 input_ids_a, input_masks_a, input_segments_a = return_id(
 answer, None, 'longest_first', max_sequence_length)
 return [input_ids_q, input_masks_q, input_segments_q,
input_ids_a, input_masks_a, input_segments_a]
def compute_input_arrays(df, columns, tokenizer, max_sequence_length):
input_ids_q, input_masks_q, input_segments_q = [], [], []
input_ids_a, input_masks_a, input_segments_a = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
ids_q, masks_q, segments_q, ids_a, masks_a, segments_a = \
_convert_to_transformer_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids_q.append(ids_q)
 input_masks_q.append(masks_q)
 input_segments_q.append(segments_q)
 input_ids_a.append(ids_a)
 input_masks_a.append(masks_a)
 input_segments_a.append(segments_a)
 return [np.asarray(input_ids_q, dtype=np.int32),
np.asarray(input_masks_q, dtype=np.int32),
np.asarray(input_segments_q, dtype=np.int32),
np.asarray(input_ids_a, dtype=np.int32),
np.asarray(input_masks_a, dtype=np.int32),
np.asarray(input_segments_a, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )
h2ws = {}
new_whale = 'new_whale'
for p, w in tagged.items() :
if w != new_whale:
h = p2h[p]
if h not in h2ws: h2ws[h] = []
if w not in h2ws[h]: h2ws[h].append(w)
 for h, ws in h2ws.items() :
if len(ws)> 1:
h2ws[h] = sorted(ws)
 w2hs = {}
for h, ws in h2ws.items() :
if len(ws)== 1:
w = ws[0]
if w not in w2hs: w2hs[w] = []
if h not in w2hs[w]: w2hs[w].append(h)
 for w, hs in w2hs.items() :
if len(hs)> 1:
w2hs[w] = sorted(hs )<categorify>
def compute_spearmanr_ignore_nan(trues, preds):
rhos = []
for tcol, pcol in zip(np.transpose(trues), np.transpose(preds)) :
rhos.append(spearmanr(tcol, pcol ).correlation)
 return np.nanmean(rhos)
 def create_model() :
q_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 config = BertConfig()
config.output_hidden_states = False
bert_model = TFBertModel.from_pretrained(
 BERT_PATH+'bert-base-uncased-tf_model.h5', config=config)
 q_embedding = bert_model(q_id, attention_mask=q_mask, token_type_ids=q_atn)[0]
a_embedding = bert_model(a_id, attention_mask=a_mask, token_type_ids=a_atn)[0]
q = tf.keras.layers.GlobalAveragePooling1D()(q_embedding)
 a = tf.keras.layers.GlobalAveragePooling1D()(a_embedding)
 x = tf.keras.layers.Concatenate()([q, a])
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(30, activation='sigmoid' )(x)
 model = tf.keras.models.Model(inputs=[q_id, q_mask, q_atn, a_id, a_mask, a_atn,], outputs=x)
 return model
train = []
for hs in w2hs.values() :
if len(hs)> 1:
train += hs
random.shuffle(train)
 train_set = set(train)
 w2ts = {}
for w, hs in w2hs.items() :
for h in hs:
if h in train_set:
if w not in w2ts:
w2ts[w] = []
if h not in w2ts[w]:
w2ts[w].append(h)
 for w, ts in w2ts.items() :
w2ts[w] = np.array(ts)
 t2i = {}
for i, t in enumerate(train):
t2i[t] = i<define_variables>
outputs = compute_output_arrays(df_train, output_categories)
 inputs = compute_input_arrays(df_train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arrays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH )
class TrainingData(Sequence):
def __init__(self, score, steps=1000, batch_size=32):

 super(TrainingData, self ).__init__()
self.score = -score
self.steps = steps
self.batch_size = batch_size
for ts in w2ts.values() :
idxs = [t2i[t] for t in ts]
for i in idxs:
for j in idxs:
self.score[
i, j] = 10000.0
self.on_epoch_end()
def __getitem__(self, index):
start = self.batch_size * index
end = min(start + self.batch_size, len(self.match)+ len(self.unmatch))
size = end - start
assert size > 0
a = np.zeros(( size,)+ img_shape, dtype=K.floatx())
 b = np.zeros(( size,)+ img_shape, dtype=K.floatx())
 c = np.zeros(( size, 1), dtype=K.floatx())
 j = start // 2
for i in range(0, size, 2):
a[i, :, :, :] = read_for_training(self.match[j][0])
 b[i, :, :, :] = read_for_training(self.match[j][1])
 c[i, 0] = 1
a[i + 1, :, :, :] = read_for_training(self.unmatch[j][0])
 b[i + 1, :, :, :] = read_for_training(self.unmatch[j][1])
 c[i + 1, 0] = 0
j += 1
return [a, b], c
def on_epoch_end(self):
if self.steps <= 0: return
self.steps -= 1
self.match = []
self.unmatch = []
_, _, x = lapjv(self.score)
 y = np.arange(len(x), dtype=np.int32)
 for ts in w2ts.values() :
d = ts.copy()
while True:
random.shuffle(d)
 if not np.any(ts == d): break
for ab in zip(ts, d): self.match.append(ab)
 for i, j in zip(x, y):
if i == j:
print(self.score)
 print(x)
 print(y)
 print(i, j)
 assert i != j
self.unmatch.append(( train[i], train[j]))
self.score[x, y] = 10000.0
self.score[y, x] = 10000.0
random.shuffle(self.match)
 random.shuffle(self.unmatch)
 assert len(self.match)== len(train)and len(self.unmatch)== len(train)
 def __len__(self):
return(len(self.match)+ len(self.unmatch)+ self.batch_size - 1)// self.batch_size
score = np.random.random_sample(size=(len(train), len(train)))
 data = TrainingData(score)
(a, b), c = data[0]<train_model>
gkf = GroupKFold(n_splits=5 ).split(X=df_train.question_body, groups=df_train.question_body)
 valid_preds = []
test_preds = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold in [0, 2]:
train_inputs = [inputs[i][train_idx] for i in range(len(inputs)) ]
train_outputs = outputs[train_idx]
valid_inputs = [inputs[i][valid_idx] for i in range(len(inputs)) ]
valid_outputs = outputs[valid_idx]
K.clear_session()
model = create_model()
optimizer = tf.keras.optimizers.Adam(learning_rate=2e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 model.fit(train_inputs, train_outputs, epochs=3, batch_size=6)
 valid_preds.append(model.predict(valid_inputs))
test_preds.append(model.predict(test_inputs))
rho_val = compute_spearmanr_ignore_nan(valid_outputs, valid_preds[-1])
 print('validation score = ', rho_val )
def set_lr(model, lr):
K.set_value(model.optimizer.lr, float(lr))
def get_lr(model):
return K.get_value(model.optimizer.lr)
 def score_reshape(score, x, y=None):

 if y is None:
m = np.zeros(( x.shape[0], x.shape[0]), dtype=K.floatx())
 m[np.triu_indices(x.shape[0], 1)] = score.squeeze()
m += m.transpose()
else:
m = np.zeros(( y.shape[0], x.shape[0]), dtype=K.floatx())
 iy, ix = np.indices(( y.shape[0], x.shape[0]))
ix = ix.reshape(( ix.size,))
iy = iy.reshape(( iy.size,))
m[iy, ix] = score.squeeze()
return m
def compute_score(verbose=1):

 features = branch_model.predict_generator(FeatureGen(train, verbose=verbose), max_queue_size=12, workers=6,
verbose=0)
 score = head_model.predict_generator(ScoreGen(features, verbose=verbose), max_queue_size=12, workers=6, verbose=0)
 score = score_reshape(score, features)
 return features, score
def make_steps(step, ampl):

 global w2ts, t2i, steps, features, score, histories
random.shuffle(train)
 w2ts = {}
for w, hs in w2hs.items() :
for h in hs:
if h in train_set:
if w not in w2ts: w2ts[w] = []
if h not in w2ts[w]: w2ts[w].append(h)
 for w, ts in w2ts.items() : w2ts[w] = np.array(ts)
 t2i = {}
for i, t in enumerate(train): t2i[t] = i
features, score = compute_score()
history = model.fit_generator(
 TrainingData(score + ampl * np.random.random_sample(size=score.shape), steps=step, batch_size=32),
initial_epoch=steps, epochs=steps + step, max_queue_size=12, workers=6, verbose=1 ).history
steps += step
history['epochs'] = steps
history['ms'] = np.mean(score)
 history['lr'] = get_lr(model)
 print(history['epochs'], history['lr'], history['ms'])
 histories.append(history )<load_pretrained>
df_sub.iloc[:, 1:] = np.average(test_preds, axis=0)
 df_sub.to_csv('submission.csv', index=False )