Kaggle competition: Natural Language Processing with Disaster Tweets
def read_cropped_image(p, augment):

 if p in h2p:
p = h2p[p]
size_x, size_y = p2size[p]
row = p2bb.loc[p]
x0, y0, x1, y1 = row['x0'], row['y0'], row['x1'], row['y1']
dx = x1 - x0
dy = y1 - y0
x0 -= dx * crop_margin
x1 += dx * crop_margin + 1
y0 -= dy * crop_margin
y1 += dy * crop_margin + 1
if x0 < 0:
x0 = 0
if x1 > size_x:
x1 = size_x
if y0 < 0:
y0 = 0
if y1 > size_y:
y1 = size_y
dx = x1 - x0
dy = y1 - y0
if dx > dy * anisotropy:
dy = 0.5 *(dx / anisotropy - dy)
 y0 -= dy
y1 += dy
else:
dx = 0.5 *(dy * anisotropy - dx)
 x0 -= dx
x1 += dx
trans = np.array([[1, 0, -0.5 * img_shape[0]], [0, 1, -0.5 * img_shape[1]], [0, 0, 1]])
 trans = np.dot(np.array([[(y1 - y0)/ img_shape[0], 0, 0], [0,(x1 - x0)/ img_shape[1], 0], [0, 0, 1]]), trans)
 if augment:
trans = np.dot(build_transform(
 random.uniform(-5, 5),
random.uniform(-5, 5),
random.uniform(0.8, 1.0),
random.uniform(0.8, 1.0),
random.uniform(-0.05 *(y1 - y0), 0.05 *(y1 - y0)) ,
random.uniform(-0.05 *(x1 - x0), 0.05 *(x1 - x0)) 
), trans)
 trans = np.dot(np.array([[1, 0, 0.5 *(y1 + y0)], [0, 1, 0.5 *(x1 + x0)], [0, 0, 1]]), trans)
 img = read_raw_image(p ).convert('L')
 img = img_to_array(img)
 matrix = trans[:2, :2]
offset = trans[:2, 2]
img = img.reshape(img.shape[:-1])
 img = affine_transform(img, matrix, offset, output_shape=img_shape[:-1], order=1, mode='constant',
cval=np.average(img))
img = img.reshape(img_shape)
 img -= np.mean(img, keepdims=True)
 img /= np.std(img, keepdims=True)+ K.epsilon()
return img
def read_for_training(p):

 return read_cropped_image(p, True)
 def read_for_validation(p):

 return read_cropped_image(p, False)
 p = list(tagged.keys())[312]<categorify>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
def subblock(x, filter, **kwargs):
x = BatchNormalization()(x)
 y = x
y = Conv2D(filter,(1, 1), activation='relu', **kwargs )(y)
 y = BatchNormalization()(y)
 y = Conv2D(filter,(3, 3), activation='relu', **kwargs )(y)
 y = BatchNormalization()(y)
 y = Conv2D(K.int_shape(x)[-1],(1, 1), **kwargs )(y)
 y = Add()([x, y])
 y = Activation('relu' )(y)
 return y
def build_model(lr, l2, activation='sigmoid'):
regul = regularizers.l2(l2)
 optim = Adam(lr=lr)
 kwargs = {'padding': 'same', 'kernel_regularizer': regul}
inp = Input(shape=img_shape)
 x = Conv2D(64,(9, 9), strides=2, activation='relu', **kwargs )(inp)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 for _ in range(2):
x = BatchNormalization()(x)
 x = Conv2D(64,(3, 3), activation='relu', **kwargs )(x)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(128,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 64, **kwargs)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(256,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 64, **kwargs)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(384,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 96, **kwargs)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(512,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 128, **kwargs)
 x = GlobalMaxPooling2D()(x)
 branch_model = Model(inp, x)
 mid = 32
xa_inp = Input(shape=branch_model.output_shape[1:])
 xb_inp = Input(shape=branch_model.output_shape[1:])
 x1 = Lambda(lambda x: x[0] * x[1] )([xa_inp, xb_inp])
 x2 = Lambda(lambda x: x[0] + x[1] )([xa_inp, xb_inp])
 x3 = Lambda(lambda x: K.abs(x[0] - x[1]))([xa_inp, xb_inp])
 x4 = Lambda(lambda x: K.square(x))(x3)
 x = Concatenate()([x1, x2, x3, x4])
 x = Reshape(( 4, branch_model.output_shape[1], 1), name='reshape1' )(x)
 x = Conv2D(mid,(4, 1), activation='relu', padding='valid' )(x)
 x = Reshape(( branch_model.output_shape[1], mid, 1))(x)
 x = Conv2D(1,(1, mid), activation='linear', padding='valid' )(x)
 x = Flatten(name='flatten' )(x)
 x = Dense(1, use_bias=True, activation=activation, name='weighted-average' )(x)
 head_model = Model([xa_inp, xb_inp], x, name='head')
 img_a = Input(shape=img_shape)
 img_b = Input(shape=img_shape)
 xa = branch_model(img_a)
 xb = branch_model(img_b)
 x = head_model([xa, xb])
 model = Model([img_a, img_b], x)
 model.compile(optim, loss='binary_crossentropy', metrics=['binary_crossentropy', 'acc'])
 return model, branch_model, head_model
model, branch_model, head_model = build_model(64e-5, 0 )<feature_engineering>
import tokenization
import matplotlib.pyplot as plt
import seaborn as sns
import re
import nltk
import spacy
import sys
import random
import fuzzywuzzy
from fuzzywuzzy import process
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from spacy.util import minibatch
from tensorflow import math
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
h2ws = {}
new_whale = 'new_whale'
for p, w in tagged.items() :
if w != new_whale:
h = p2h[p]
if h not in h2ws: h2ws[h] = []
if w not in h2ws[h]: h2ws[h].append(w)
 for h, ws in h2ws.items() :
if len(ws)> 1:
h2ws[h] = sorted(ws)
 w2hs = {}
for h, ws in h2ws.items() :
if len(ws)== 1:
w = ws[0]
if w not in w2hs: w2hs[w] = []
if h not in w2hs[w]: w2hs[w].append(h)
 for w, hs in w2hs.items() :
if len(hs)> 1:
w2hs[w] = sorted(hs )<categorify>
random_seed = 0
data = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 print(data.shape)
 data.head()
train = []
for hs in w2hs.values() :
if len(hs)> 1:
train += hs
random.shuffle(train)
 train_set = set(train)
 w2ts = {}
for w, hs in w2hs.items() :
for h in hs:
if h in train_set:
if w not in w2ts:
w2ts[w] = []
if h not in w2ts[w]:
w2ts[w].append(h)
 for w, ts in w2ts.items() :
w2ts[w] = np.array(ts)
 t2i = {}
for i, t in enumerate(train):
t2i[t] = i<define_variables>
print(data.isnull().sum())
 display(data.nunique() )
class TrainingData(Sequence):
def __init__(self, score, steps=1000, batch_size=32):

 super(TrainingData, self ).__init__()
self.score = -score
self.steps = steps
self.batch_size = batch_size
for ts in w2ts.values() :
idxs = [t2i[t] for t in ts]
for i in idxs:
for j in idxs:
self.score[
i, j] = 10000.0
self.on_epoch_end()
def __getitem__(self, index):
start = self.batch_size * index
end = min(start + self.batch_size, len(self.match)+ len(self.unmatch))
size = end - start
assert size > 0
a = np.zeros(( size,)+ img_shape, dtype=K.floatx())
 b = np.zeros(( size,)+ img_shape, dtype=K.floatx())
 c = np.zeros(( size, 1), dtype=K.floatx())
 j = start // 2
for i in range(0, size, 2):
a[i, :, :, :] = read_for_training(self.match[j][0])
 b[i, :, :, :] = read_for_training(self.match[j][1])
 c[i, 0] = 1
a[i + 1, :, :, :] = read_for_training(self.unmatch[j][0])
 b[i + 1, :, :, :] = read_for_training(self.unmatch[j][1])
 c[i + 1, 0] = 0
j += 1
return [a, b], c
def on_epoch_end(self):
if self.steps <= 0: return
self.steps -= 1
self.match = []
self.unmatch = []
_, _, x = lapjv(self.score)
 y = np.arange(len(x), dtype=np.int32)
 for ts in w2ts.values() :
d = ts.copy()
while True:
random.shuffle(d)
 if not np.any(ts == d): break
for ab in zip(ts, d): self.match.append(ab)
 for i, j in zip(x, y):
if i == j:
print(self.score)
 print(x)
 print(y)
 print(i, j)
 assert i != j
self.unmatch.append(( train[i], train[j]))
self.score[x, y] = 10000.0
self.score[y, x] = 10000.0
random.shuffle(self.match)
 random.shuffle(self.unmatch)
 assert len(self.match)== len(train)and len(self.unmatch)== len(train)
 def __len__(self):
return(len(self.match)+ len(self.unmatch)+ self.batch_size - 1)// self.batch_size
score = np.random.random_sample(size=(len(train), len(train)))
 data = TrainingData(score)
(a, b), c = data[0]<train_model>
def utils_preprocess_text(text, flg_stemm=False, flg_lemm=True, lst_stopwords=None):
text = re.sub(r"https?://\S+|www\.\S+", "", text)
 html = re.compile(r"<.*?>|&([a-z0-9]+|
text = re.sub(html, "", text)
 l = len(text)
 t = text
text= re.sub(r'[^\x00-\x7f]',r'', text)
 text = re.sub(r'[^\w\s]', '', str(text ).lower().strip())
 lst_text = text.split()
if lst_stopwords is not None:
lst_text = [word for word in lst_text if word not in
lst_stopwords]
if flg_stemm == True:
ps = nltk.stem.porter.PorterStemmer()
lst_text = [ps.stem(word)for word in lst_text]
if flg_lemm == True:
lem = nltk.stem.wordnet.WordNetLemmatizer()
lst_text = [lem.lemmatize(word)for word in lst_text]
text = " ".join(lst_text)
 return text
def set_lr(model, lr):
K.set_value(model.optimizer.lr, float(lr))
def get_lr(model):
return K.get_value(model.optimizer.lr)
 def score_reshape(score, x, y=None):

 if y is None:
m = np.zeros(( x.shape[0], x.shape[0]), dtype=K.floatx())
 m[np.triu_indices(x.shape[0], 1)] = score.squeeze()
m += m.transpose()
else:
m = np.zeros(( y.shape[0], x.shape[0]), dtype=K.floatx())
 iy, ix = np.indices(( y.shape[0], x.shape[0]))
ix = ix.reshape(( ix.size,))
iy = iy.reshape(( iy.size,))
m[iy, ix] = score.squeeze()
return m
def compute_score(verbose=1):

 features = branch_model.predict_generator(FeatureGen(train, verbose=verbose), max_queue_size=12, workers=6,
verbose=0)
 score = head_model.predict_generator(ScoreGen(features, verbose=verbose), max_queue_size=12, workers=6, verbose=0)
 score = score_reshape(score, features)
 return features, score
def make_steps(step, ampl):

 global w2ts, t2i, steps, features, score, histories
random.shuffle(train)
 w2ts = {}
for w, hs in w2hs.items() :
for h in hs:
if h in train_set:
if w not in w2ts: w2ts[w] = []
if h not in w2ts[w]: w2ts[w].append(h)
 for w, ts in w2ts.items() : w2ts[w] = np.array(ts)
 t2i = {}
for i, t in enumerate(train): t2i[t] = i
features, score = compute_score()
history = model.fit_generator(
 TrainingData(score + ampl * np.random.random_sample(size=score.shape), steps=step, batch_size=32),
initial_epoch=steps, epochs=steps + step, max_queue_size=12, workers=6, verbose=1 ).history
steps += step
history['epochs'] = steps
history['ms'] = np.mean(score)
 history['lr'] = get_lr(model)
 print(history['epochs'], history['lr'], history['ms'])
 histories.append(history )<load_pretrained>
lst_stopwords = nltk.corpus.stopwords.words("english")
 data["text_clean"] = data["text"].apply(lambda x: utils_preprocess_text(x, flg_stemm=False, flg_lemm=True, lst_stopwords=lst_stopwords))
data.head()
def prepare_submission(threshold, filename):

 vtop = 0
vhigh = 0
pos = [0, 0, 0, 0, 0, 0]
with open(filename, 'wt', newline='
')as f:
f.write('Image,Id
')
 for i, p in enumerate(tqdm(submit)) :
t = []
s = set()
a = score[i, :]
for j in list(reversed(np.argsort(a))):
h = known[j]
if a[j] < threshold and new_whale not in s:
pos[len(t)] += 1
s.add(new_whale)
 t.append(new_whale)
 if len(t)== 5: break;
for w in h2ws[h]:
assert w != new_whale
if w not in s:
if a[j] > 1.0:
vtop += 1
elif a[j] >= threshold:
vhigh += 1
s.add(w)
 t.append(w)
 if len(t)== 5: break;
if len(t)== 5: break;
if new_whale not in s: pos[5] += 1
assert len(t)== 5 and len(s)== 5
f.write(p + ',' + ' '.join(t[:5])+ '
')
 return vtop, vhigh, pos<load_pretrained>
def clean_location(data):
data['location'] = data['location'].str.lower()
data['location'] = data['location'].str.strip()
data['location'] = data['location'].apply(lambda x: re.sub(r',(?s ).*$', r'', x)if str(x)!= str(np.nan)else np.nan)
 return data
histories = []
steps = 0
if isfile('.. /input/piotte/mpiotte-standard.model'):
tmp = keras.models.load_model('.. /input/piotte/mpiotte-standard.model')
 model.set_weights(tmp.get_weights())
 tic = time.time()
h2ws = {}
for p, w in tagged.items() :
if w != new_whale:
h = p2h[p]
if h not in h2ws: h2ws[h] = []
if w not in h2ws[h]: h2ws[h].append(w)
 known = sorted(list(h2ws.keys()))
h2i = {}
for i, h in enumerate(known): h2i[h] = i
fknown1 = branch_model.predict_generator(FeatureGen(known), max_queue_size=20, workers=10, verbose=1)
 fsubmit1 = branch_model.predict_generator(FeatureGen(submit), max_queue_size=20, workers=10, verbose=1)
 score1 = head_model.predict_generator(ScoreGen(fknown1, fsubmit1), max_queue_size=20, workers=10, verbose=1)
 score1 = score_reshape(score1, fknown1, fsubmit1)
 tmp = keras.models.load_model('.. /input/piotte/mpiotte-bootstrap.model')
 model.set_weights(tmp.get_weights())
 tic = time.time()
h2ws = {}
for p, w in tagged.items() :
if w != new_whale:
h = p2h[p]
if h not in h2ws: h2ws[h] = []
if w not in h2ws[h]: h2ws[h].append(w)
 known = sorted(list(h2ws.keys()))
h2i = {}
for i, h in enumerate(known): h2i[h] = i
fknown2 = branch_model.predict_generator(FeatureGen(known), max_queue_size=20, workers=10, verbose=1)
 fsubmit2 = branch_model.predict_generator(FeatureGen(submit), max_queue_size=20, workers=10, verbose=1)
 score2 = head_model.predict_generator(ScoreGen(fknown2, fsubmit2), max_queue_size=20, workers=10, verbose=1)
 score2 = score_reshape(score2, fknown2, fsubmit2)
 else:
make_steps(10, 1000)
 ampl = 100.0
for _ in range(2):
print('noise ampl.= ', ampl)
 make_steps(5, ampl)
 ampl = max(1.0, 100 ** -0.1 * ampl)
 for _ in range(18): make_steps(5, 1.0)
 set_lr(model, 16e-5)
 for _ in range(10): make_steps(5, 0.5)
 set_lr(model, 4e-5)
 for _ in range(8): make_steps(5, 0.25)
 set_lr(model, 1e-5)
 for _ in range(2): make_steps(5, 0.25)
 weights = model.get_weights()
model, branch_model, head_model = build_model(64e-5,0.0002)
 model.set_weights(weights)
 for _ in range(10): make_steps(5, 1.0)
 set_lr(model, 16e-5)
 for _ in range(10): make_steps(5, 0.5)
 set_lr(model, 4e-5)
 for _ in range(8): make_steps(5, 0.25)
 set_lr(model, 1e-5)
 for _ in range(2): make_steps(5, 0.25 )<compute_test_metric>
data = clean_location(data )
score = 0.45*score1 + 0.55*score2<train_model>
def replace_matches_in_column(df, column, string_to_match, min_ratio = 90):
strings = df[column].unique()
matches = fuzzywuzzy.process.extract(string_to_match, strings, limit=10, scorer=fuzzywuzzy.fuzz.token_sort_ratio)
 close_matches = [matches[0] for matches in matches if matches[1] >= min_ratio]
rows_with_matches = df[column].isin(close_matches)
 df.loc[rows_with_matches, column] = string_to_match
prepare_submission(0.92, 'submission_0.45_standard_0.55_boostrap-400.csv')
 toc = time.time()
print("Submission time: ",(toc - tic)/ 60.)<import_modules>
locations = data.groupby('location' ).location.count().sort_values(ascending=False)
 locations = locations[locations.values > 3]
locations = locations.index
for loc in locations:
replace_matches_in_column(df=data, column='location', string_to_match=loc )
!pip install lapjv==1.3.1
Lambda, MaxPooling2D, Reshape 
<load_from_csv>
def rename_location(data):
data.loc[(data.location == 'united states'), 'location'] = 'usa'
data.loc[(data.location == 'united states of america'), 'location'] = 'usa'
data.loc[(data.location == 'us'), 'location'] = 'usa'
data.loc[(data.location == 'u.s.'), 'location'] = 'usa'
data.loc[(data.location == 'u.s.a'), 'location'] = 'usa'
data.loc[(data.location == 'u.s.a.'), 'location'] = 'usa'
data.loc[(data.location == 'new york'), 'location'] = 'nyc'
data.loc[(data.location == 'ny'), 'location'] = 'nyc'
data.loc[(data.location == 'new york city'), 'location'] = 'nyc'
data.loc[(data.location == 'uk'), 'location'] = 'england'
data.loc[(data.location == 'uk.'), 'location'] = 'england'
data.loc[(data.location == 'united kingdom'), 'location'] = 'england'
data.loc[(data.location == '?? | pittsburgh'), 'location'] = 'pittsburgh'
data.loc[(data.location == '??????'), 'location'] = np.nan
data.loc[(data.location == '?????'), 'location'] = np.nan
data.loc[(data.location == '??'), 'location'] = np.nan
data.loc[(data.location == 'world'), 'location'] = np.nan
data.loc[(data.location == 'world wide'), 'location'] = np.nan
data.loc[(data.location == 'worldwide'), 'location'] = np.nan
data.loc[(data.location == 'everywhere'), 'location'] = np.nan
data.loc[(data.location == 'everywhere.. '), 'location'] = np.nan
data.loc[(data.location == 'earth'), 'location'] = np.nan
data.loc[(data.location == 'happily married with 2 kids'), 'location'] = np.nan
data.loc[(data.location == 'road to the billionaires club'), 'location'] = np.nan
data.loc[(data.location == 'pedophile hunting ground'), 'location'] = np.nan
data.loc[(data.location == 'does it really matter!'), 'location'] = np.nan
data.loc[(data.location == 'planet earth'), 'location'] = np.nan
data.loc[(data.location == 'in the word of god'), 'location'] = np.nan
data.loc[(data.location == 'mad as hell'), 'location'] = np.nan
data.loc[(data.location == 'global'), 'location'] = np.nan
data.loc[(data.location == '
data.loc[(data.location == '304'), 'location'] = np.nan
data.loc[(data.location == ''), 'location'] = np.nan
return data
TRAIN_DF = '.. /input/humpback-whale-identification/train.csv'
SUB_Df = '.. /input/humpback-whale-identification/sample_submission.csv'
TRAIN = '.. /input/humpback-whale-identification/train/'
TEST = '.. /input/humpback-whale-identification/test/'
P2H = '.. /input/metadata/p2h.pickle'
P2SIZE = '.. /input/metadata/p2size.pickle'
BB_DF = ".. /input/metadata/bounding_boxes.csv"
tagged = dict([(p, w)for _, p, w in read_csv(TRAIN_DF ).to_records() ])
 submit = [p for _, p, _ in read_csv(SUB_Df ).to_records() ]
join = list(tagged.keys())+ submit<train_model>
locations = data.groupby('location' ).location.count().sort_values(ascending=False)
 locations = locations[locations.values <= 3]
locations = locations.index
for loc in locations:
data.loc[(data.location == loc), 'location'] = np.nan
data.groupby('location' ).location.count().sort_values(ascending=False )
if isfile(P2SIZE):
print("P2SIZE exists.")
 with open(P2SIZE, 'rb')as f:
p2size = pickle.load(f)
 else:
p2size = {}
for p in tqdm(join):
size = pil_image.open(expand_path(p)).size
p2size[p] = size<compute_test_metric>
def concatenate(data):
data['sequence'] = data['text_clean'].map(str)+ ' XXLOC ' + data['location'].map(str)\
+ ' XXKEY ' + data['keyword'].map(str)+ ' TWTXX'
return data
data = concatenate(data)
 print(data.sequence[0])
 print(data.sequence[98])
 print(data.sequence[100] )
def match(h1, h2):
for p1 in h2ps[h1]:
for p2 in h2ps[h2]:
i1 = pil_image.open(expand_path(p1))
i2 = pil_image.open(expand_path(p2))
if i1.mode != i2.mode or i1.size != i2.size: return False
a1 = np.array(i1)
 a1 = a1 - a1.mean()
a1 = a1 / sqrt(( a1 ** 2 ).mean())
 a2 = np.array(i2)
 a2 = a2 - a2.mean()
a2 = a2 / sqrt(( a2 ** 2 ).mean())
 a =(( a1 - a2)** 2 ).mean()
if a > 0.1: return False
return True
if isfile(P2H):
print("P2H exists.")
 with open(P2H, 'rb')as f:
p2h = pickle.load(f)
 else:
p2h = {}
for p in tqdm(join):
img = pil_image.open(expand_path(p))
h = phash(img)
 p2h[p] = h
h2ps = {}
for p, h in p2h.items() :
if h not in h2ps: h2ps[h] = []
if p not in h2ps[h]: h2ps[h].append(p)
 hs = list(h2ps.keys())
 h2h = {}
for i, h1 in enumerate(tqdm(hs)) :
for h2 in hs[:i]:
if h1 - h2 <= 6 and match(h1, h2):
s1 = str(h1)
 s2 = str(h2)
 if s1 < s2: s1, s2 = s2, s1
h2h[s1] = s2
for p, h in p2h.items() :
h = str(h)
 if h in h2h: h = h2h[h]
p2h[p] = h
h2ps = {}
for p, h in p2h.items() :
if h not in h2ps: h2ps[h] = []
if p not in h2ps[h]: h2ps[h].append(p )<find_best_params>
test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 test["text_clean"] = test["text"].apply(lambda x: utils_preprocess_text(x, flg_stemm=False, flg_lemm=True, lst_stopwords=lst_stopwords))
test = clean_location(test)
 locations = test.groupby('location' ).location.count().sort_values(ascending=False)
 locations = locations.index
for loc in locations:
replace_matches_in_column(df=test, column='location', string_to_match=loc)
 locations = data.groupby('location' ).location.count().sort_values(ascending=False)
 locations = locations.index
for loc in locations:
replace_matches_in_column(df=test, column='location', string_to_match=loc)
 test = rename_location(test)
 test = concatenate(test )
def prefer(ps):
if len(ps)== 1: return ps[0]
best_p = ps[0]
best_s = p2size[best_p]
for i in range(1, len(ps)) :
p = ps[i]
s = p2size[p]
if s[0] * s[1] > best_s[0] * best_s[1]:
best_p = p
best_s = s
return best_p
h2p = {}
for h, ps in h2ps.items() :
h2p[h] = prefer(ps)
 len(h2p), list(h2p.items())[:5] 
<set_options>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
p2bb = pd.read_csv(BB_DF ).set_index("Image")
 old_stderr = sys.stderr
sys.stderr = open('/dev/null' if platform.system() != 'Windows' else 'nul', 'w')
 sys.stderr = old_stderr
img_shape =(384, 384, 1)
 anisotropy = 2.15
crop_margin = 0.05<normalization>
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model
def build_transform(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):

 rotation = np.deg2rad(rotation)
 shear = np.deg2rad(shear)
 rotation_matrix = np.array(
 [[np.cos(rotation), np.sin(rotation), 0], [-np.sin(rotation), np.cos(rotation), 0], [0, 0, 1]])
 shift_matrix = np.array([[1, 0, height_shift], [0, 1, width_shift], [0, 0, 1]])
 shear_matrix = np.array([[1, np.sin(shear), 0], [0, np.cos(shear), 0], [0, 0, 1]])
 zoom_matrix = np.array([[1.0 / height_zoom, 0, 0], [0, 1.0 / width_zoom, 0], [0, 0, 1]])
 shift_matrix = np.array([[1, 0, -height_shift], [0, 1, -width_shift], [0, 0, 1]])
 return np.dot(np.dot(rotation_matrix, shear_matrix), np.dot(zoom_matrix, shift_matrix))<prepare_x_and_y>
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
def read_cropped_image(p, augment):

 if p in h2p:
p = h2p[p]
size_x, size_y = p2size[p]
row = p2bb.loc[p]
x0, y0, x1, y1 = row['x0'], row['y0'], row['x1'], row['y1']
dx = x1 - x0
dy = y1 - y0
x0 -= dx * crop_margin
x1 += dx * crop_margin + 1
y0 -= dy * crop_margin
y1 += dy * crop_margin + 1
if x0 < 0:
x0 = 0
if x1 > size_x:
x1 = size_x
if y0 < 0:
y0 = 0
if y1 > size_y:
y1 = size_y
dx = x1 - x0
dy = y1 - y0
if dx > dy * anisotropy:
dy = 0.5 *(dx / anisotropy - dy)
 y0 -= dy
y1 += dy
else:
dx = 0.5 *(dy * anisotropy - dx)
 x0 -= dx
x1 += dx
trans = np.array([[1, 0, -0.5 * img_shape[0]], [0, 1, -0.5 * img_shape[1]], [0, 0, 1]])
 trans = np.dot(np.array([[(y1 - y0)/ img_shape[0], 0, 0], [0,(x1 - x0)/ img_shape[1], 0], [0, 0, 1]]), trans)
 if augment:
trans = np.dot(build_transform(
 random.uniform(-5, 5),
random.uniform(-5, 5),
random.uniform(0.8, 1.0),
random.uniform(0.8, 1.0),
random.uniform(-0.05 *(y1 - y0), 0.05 *(y1 - y0)) ,
random.uniform(-0.05 *(x1 - x0), 0.05 *(x1 - x0)) 
), trans)
 trans = np.dot(np.array([[1, 0, 0.5 *(y1 + y0)], [0, 1, 0.5 *(x1 + x0)], [0, 0, 1]]), trans)
 img = read_raw_image(p ).convert('L')
 img = img_to_array(img)
 matrix = trans[:2, :2]
offset = trans[:2, 2]
img = img.reshape(img.shape[:-1])
 img = affine_transform(img, matrix, offset, output_shape=img_shape[:-1], order=1, mode='constant',
cval=np.average(img))
img = img.reshape(img_shape)
 img -= np.mean(img, keepdims=True)
 img /= np.std(img, keepdims=True)+ K.epsilon()
return img
def read_for_training(p):

 return read_cropped_image(p, True)
 def read_for_validation(p):

 return read_cropped_image(p, False)
 p = list(tagged.keys())[312] 
<categorify>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
def subblock(x, filter, **kwargs):
x = BatchNormalization()(x)
 y = x
y = Conv2D(filter,(1, 1), activation='relu', **kwargs )(y)
 y = BatchNormalization()(y)
 y = Conv2D(filter,(3, 3), activation='relu', **kwargs )(y)
 y = BatchNormalization()(y)
 y = Conv2D(K.int_shape(x)[-1],(1, 1), **kwargs )(y)
 y = Add()([x, y])
 y = Activation('relu' )(y)
 return y
def build_model(lr, l2, activation='sigmoid'):
regul = regularizers.l2(l2)
 optim = Adam(lr=lr)
 kwargs = {'padding': 'same', 'kernel_regularizer': regul}
inp = Input(shape=img_shape)
 x = Conv2D(64,(9, 9), strides=2, activation='relu', **kwargs )(inp)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 for _ in range(2):
x = BatchNormalization()(x)
 x = Conv2D(64,(3, 3), activation='relu', **kwargs )(x)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(128,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 64, **kwargs)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(256,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 64, **kwargs)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(384,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 96, **kwargs)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(512,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 128, **kwargs)
 x = GlobalMaxPooling2D()(x)
 branch_model = Model(inp, x)
 mid = 32
xa_inp = Input(shape=branch_model.output_shape[1:])
 xb_inp = Input(shape=branch_model.output_shape[1:])
 x1 = Lambda(lambda x: x[0] * x[1] )([xa_inp, xb_inp])
 x2 = Lambda(lambda x: x[0] + x[1] )([xa_inp, xb_inp])
 x3 = Lambda(lambda x: K.abs(x[0] - x[1]))([xa_inp, xb_inp])
 x4 = Lambda(lambda x: K.square(x))(x3)
 x = Concatenate()([x1, x2, x3, x4])
 x = Reshape(( 4, branch_model.output_shape[1], 1), name='reshape1' )(x)
 x = Conv2D(mid,(4, 1), activation='relu', padding='valid' )(x)
 x = Reshape(( branch_model.output_shape[1], mid, 1))(x)
 x = Conv2D(1,(1, mid), activation='linear', padding='valid' )(x)
 x = Flatten(name='flatten' )(x)
 x = Dense(1, use_bias=True, activation=activation, name='weighted-average' )(x)
 head_model = Model([xa_inp, xb_inp], x, name='head')
 img_a = Input(shape=img_shape)
 img_b = Input(shape=img_shape)
 xa = branch_model(img_a)
 xb = branch_model(img_b)
 x = head_model([xa, xb])
 model = Model([img_a, img_b], x)
 model.compile(optim, loss='binary_crossentropy', metrics=['binary_crossentropy', 'acc'])
 return model, branch_model, head_model
model, branch_model, head_model = build_model(64e-5, 0 )<feature_engineering>
train_input = bert_encode(data.sequence.values, tokenizer, max_len=65)
 test_input = bert_encode(test.sequence.values, tokenizer, max_len=65)
 train_labels = data.target.values
h2ws = {}
new_whale = 'new_whale'
for p, w in tagged.items() :
if w != new_whale:
h = p2h[p]
if h not in h2ws: h2ws[h] = []
if w not in h2ws[h]: h2ws[h].append(w)
 for h, ws in h2ws.items() :
if len(ws)> 1:
h2ws[h] = sorted(ws)
 w2hs = {}
for h, ws in h2ws.items() :
if len(ws)== 1:
w = ws[0]
if w not in w2hs: w2hs[w] = []
if h not in w2hs[w]: w2hs[w].append(h)
 for w, hs in w2hs.items() :
if len(hs)> 1:
w2hs[w] = sorted(hs )<categorify>
checkpoint = ModelCheckpoint('model.h5', monitor='val_loss', save_best_only=True)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.15,
epochs=3,
callbacks=[checkpoint],
batch_size=8
)
train = []
for hs in w2hs.values() :
if len(hs)> 1:
train += hs
random.shuffle(train)
 train_set = set(train)
 w2ts = {}
for w, hs in w2hs.items() :
for h in hs:
if h in train_set:
if w not in w2ts:
w2ts[w] = []
if h not in w2ts[w]:
w2ts[w].append(h)
 for w, ts in w2ts.items() :
w2ts[w] = np.array(ts)
 t2i = {}
for i, t in enumerate(train):
t2i[t] = i<define_variables>
model.load_weights('model.h5')
 test_pred = model.predict(test_input )
class TrainingData(Sequence):
def __init__(self, score, steps=1000, batch_size=32):

 super(TrainingData, self ).__init__()
self.score = -score
self.steps = steps
self.batch_size = batch_size
for ts in w2ts.values() :
idxs = [t2i[t] for t in ts]
for i in idxs:
for j in idxs:
self.score[
i, j] = 10000.0
self.on_epoch_end()
def __getitem__(self, index):
start = self.batch_size * index
end = min(start + self.batch_size, len(self.match)+ len(self.unmatch))
size = end - start
assert size > 0
a = np.zeros(( size,)+ img_shape, dtype=K.floatx())
 b = np.zeros(( size,)+ img_shape, dtype=K.floatx())
 c = np.zeros(( size, 1), dtype=K.floatx())
 j = start // 2
for i in range(0, size, 2):
a[i, :, :, :] = read_for_training(self.match[j][0])
 b[i, :, :, :] = read_for_training(self.match[j][1])
 c[i, 0] = 1
a[i + 1, :, :, :] = read_for_training(self.unmatch[j][0])
 b[i + 1, :, :, :] = read_for_training(self.unmatch[j][1])
 c[i + 1, 0] = 0
j += 1
return [a, b], c
def on_epoch_end(self):
if self.steps <= 0: return
self.steps -= 1
self.match = []
self.unmatch = []
num_threads = 6
tmp = num_threads*[None]
threads = []
thread_input = num_threads*[None]
thread_idx = 0
batch = score.shape[0] //(num_threads-1)
 for start in range(0, score.shape[0], batch):
end = min(score.shape[0], start + batch)
 thread_input[thread_idx] = self.score[start:end, start:end]
thread_idx += 1
def worker(data_idx):
x,_,_ = lapjv(thread_input[data_idx])
 tmp[data_idx] = x
print("Start worker threads")
 for i in range(num_threads):
t = threading.Thread(target=worker, args=(i,), daemon=True)
 t.start()
threads.append(t)
 for t in threads:
if t is not None:
t.join()
x = np.concatenate(tmp)
 print("LAP completed")
 y = np.arange(len(x), dtype=np.int32)
 for ts in w2ts.values() :
d = ts.copy()
while True:
random.shuffle(d)
 if not np.any(ts == d): break
for ab in zip(ts, d): self.match.append(ab)
 for i, j in zip(x, y):
if i == j:
print(self.score)
 print(x)
 print(y)
 print(i, j)
 assert i != j
self.unmatch.append(( train[i], train[j]))
self.score[x, y] = 10000.0
self.score[y, x] = 10000.0
random.shuffle(self.match)
 random.shuffle(self.unmatch)
 assert len(self.match)== len(train)and len(self.unmatch)== len(train)
 def __len__(self):
return(len(self.match)+ len(self.unmatch)+ self.batch_size - 1)// self.batch_size
def __init__(self, score, steps=1000, batch_size=32):

 super(TrainingData, self ).__init__()
self.score = -score
self.steps = steps
self.batch_size = batch_size
for ts in w2ts.values() :
idxs = [t2i[t] for t in ts]
for i in idxs:
for j in idxs:
self.score[
i, j] = 10000.0
self.on_epoch_end()
def __getitem__(self, index):
start = self.batch_size * index
end = min(start + self.batch_size, len(self.match)+ len(self.unmatch))
size = end - start
assert size > 0
a = np.zeros(( size,)+ img_shape, dtype=K.floatx())
 b = np.zeros(( size,)+ img_shape, dtype=K.floatx())
 c = np.zeros(( size, 1), dtype=K.floatx())
 j = start // 2
for i in range(0, size, 2):
a[i, :, :, :] = read_for_training(self.match[j][0])
 b[i, :, :, :] = read_for_training(self.match[j][1])
 c[i, 0] = 1
a[i + 1, :, :, :] = read_for_training(self.unmatch[j][0])
 b[i + 1, :, :, :] = read_for_training(self.unmatch[j][1])
 c[i + 1, 0] = 0
j += 1
return [a, b], c
def on_epoch_end(self):
if self.steps <= 0: return
self.steps -= 1
self.match = []
self.unmatch = []
num_threads = 6
tmp = num_threads*[None]
threads = []
thread_input = num_threads*[None]
thread_idx = 0
batch = score.shape[0] //(num_threads-1)
 for start in range(0, score.shape[0], batch):
end = min(score.shape[0], start + batch)
 thread_input[thread_idx] = self.score[start:end, start:end]
thread_idx += 1
def worker(data_idx):
x,_,_ = lapjv(thread_input[data_idx])
 tmp[data_idx] = x
print("Start worker threads")
 for i in range(num_threads):
t = threading.Thread(target=worker, args=(i,), daemon=True)
 t.start()
threads.append(t)
 for t in threads:
if t is not None:
t.join()
x = np.concatenate(tmp)
 print("LAP completed")
 y = np.arange(len(x), dtype=np.int32)
 for ts in w2ts.values() :
d = ts.copy()
while True:
random.shuffle(d)
 if not np.any(ts == d): break
for ab in zip(ts, d): self.match.append(ab)
 for i, j in zip(x, y):
if i == j:
print(self.score)
 print(x)
 print(y)
 print(i, j)
 assert i != j
self.unmatch.append(( train[i], train[j]))
self.score[x, y] = 10000.0
self.score[y, x] = 10000.0
random.shuffle(self.match)
 random.shuffle(self.unmatch)
 assert len(self.match)== len(train)and len(self.unmatch)== len(train)
 def __len__(self):
return(len(self.match)+ len(self.unmatch)+ self.batch_size - 1)// self.batch_size
def __init__(self, score, steps=1000, batch_size=32):

 super(TrainingData, self ).__init__()
self.score = -score
self.steps = steps
self.batch_size = batch_size
for ts in w2ts.values() :
idxs = [t2i[t] for t in ts]
for i in idxs:
for j in idxs:
self.score[
i, j] = 10000.0
self.on_epoch_end()
def __getitem__(self, index):
start = self.batch_size * index
end = min(start + self.batch_size, len(self.match)+ len(self.unmatch))
size = end - start
assert size > 0
a = np.zeros(( size,)+ img_shape, dtype=K.floatx())
 b = np.zeros(( size,)+ img_shape, dtype=K.floatx())
 c = np.zeros(( size, 1), dtype=K.floatx())
 j = start // 2
for i in range(0, size, 2):
a[i, :, :, :] = read_for_training(self.match[j][0])
 b[i, :, :, :] = read_for_training(self.match[j][1])
 c[i, 0] = 1
a[i + 1, :, :, :] = read_for_training(self.unmatch[j][0])
 b[i + 1, :, :, :] = read_for_training(self.unmatch[j][1])
 c[i + 1, 0] = 0
j += 1
return [a, b], c
def on_epoch_end(self):
if self.steps <= 0: return
self.steps -= 1
self.match = []
self.unmatch = []
num_threads = 6
tmp = num_threads*[None]
threads = []
thread_input = num_threads*[None]
thread_idx = 0
batch = score.shape[0] //(num_threads-1)
 for start in range(0, score.shape[0], batch):
end = min(score.shape[0], start + batch)
 thread_input[thread_idx] = self.score[start:end, start:end]
thread_idx += 1
def worker(data_idx):
x,_,_ = lapjv(thread_input[data_idx])
 tmp[data_idx] = x
print("Start worker threads")
 for i in range(num_threads):
t = threading.Thread(target=worker, args=(i,), daemon=True)
 t.start()
threads.append(t)
 for t in threads:
if t is not None:
t.join()
x = np.concatenate(tmp)
 print("LAP completed")
 y = np.arange(len(x), dtype=np.int32)
 for ts in w2ts.values() :
d = ts.copy()
while True:
random.shuffle(d)
 if not np.any(ts == d): break
for ab in zip(ts, d): self.match.append(ab)
 for i, j in zip(x, y):
if i == j:
print(self.score)
 print(x)
 print(y)
 print(i, j)
 assert i != j
self.unmatch.append(( train[i], train[j]))
self.score[x, y] = 10000.0
self.score[y, x] = 10000.0
random.shuffle(self.match)
 random.shuffle(self.unmatch)
 assert len(self.match)== len(train)and len(self.unmatch)== len(train)
 def __len__(self):
return(len(self.match)+ len(self.unmatch)+ self.batch_size - 1)// self.batch_size 
<train_model>
submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 submission['target'] = test_pred.round().astype(int)
 submission.to_csv('submission.csv', index=False )