Kaggle competition: Humpback Whale Identification
<SOS> metric: MAP@{K} Kaggle data source: humpback-whale-identification<define_variables>
!pip install lapjv==1.3.1
Lambda, MaxPooling2D, Reshape
numericals = ["square_feet", "year_built", "air_temperature", "cloud_coverage",
"dew_temperature", 'precip_depth_1_hr', 'floor_count', 'beaufort_scale','air_temperature_mean_lag72', 'air_temperature_max_lag72',
'air_temperature_min_lag72', 'air_temperature_std_lag72']
categoricals = ["site_id", "building_id", "primary_use", "hour", "weekday", "meter", "wind_direction"]
target = 'meter_reading'
feat_cols = categoricals + numericals<feature_engineering>
TRAIN_DF = '.. /input/humpback-whale-identification/train.csv'
SUB_Df = '.. /input/humpback-whale-identification/sample_submission.csv'
TRAIN = '.. /input/humpback-whale-identification/train/'
TEST = '.. /input/humpback-whale-identification/test/'
P2H = '.. /input/metadata/p2h.pickle'
P2SIZE = '.. /input/metadata/p2size.pickle'
BB_DF = ".. /input/metadata/bounding_boxes.csv"
tagged = dict([(p, w)for _, p, w in read_csv(TRAIN_DF ).to_records() ])
 submit = [p for _, p, _ in read_csv(SUB_Df ).to_records() ]
join = list(tagged.keys())+ submit
df_train[target] = np.log1p(df_train[target] )<prepare_x_and_y>
if isfile(P2SIZE):
print("P2SIZE exists.")
 with open(P2SIZE, 'rb')as f:
p2size = pickle.load(f)
 else:
p2size = {}
for p in tqdm(join):
size = pil_image.open(expand_path(p)).size
p2size[p] = size
X_half_1 = df_train.loc[df_train.group==1][feat_cols]
X_half_2 = df_train.loc[df_train.group==2][feat_cols]
y_half_1 = df_train.loc[df_train.group==1][target]
y_half_2 = df_train.loc[df_train.group==2][target]<create_dataframe>
def match(h1, h2):
for p1 in h2ps[h1]:
for p2 in h2ps[h2]:
i1 = pil_image.open(expand_path(p1))
i2 = pil_image.open(expand_path(p2))
if i1.mode != i2.mode or i1.size != i2.size: return False
a1 = np.array(i1)
 a1 = a1 - a1.mean()
a1 = a1 / sqrt(( a1 ** 2 ).mean())
 a2 = np.array(i2)
 a2 = a2 - a2.mean()
a2 = a2 / sqrt(( a2 ** 2 ).mean())
 a =(( a1 - a2)** 2 ).mean()
if a > 0.1: return False
return True
if isfile(P2H):
print("P2H exists.")
 with open(P2H, 'rb')as f:
p2h = pickle.load(f)
 else:
p2h = {}
for p in tqdm(join):
img = pil_image.open(expand_path(p))
h = phash(img)
 p2h[p] = h
h2ps = {}
for p, h in p2h.items() :
if h not in h2ps: h2ps[h] = []
if p not in h2ps[h]: h2ps[h].append(p)
 hs = list(h2ps.keys())
 h2h = {}
for i, h1 in enumerate(tqdm(hs)) :
for h2 in hs[:i]:
if h1 - h2 <= 6 and match(h1, h2):
s1 = str(h1)
 s2 = str(h2)
 if s1 < s2: s1, s2 = s2, s1
h2h[s1] = s2
for p, h in p2h.items() :
h = str(h)
 if h in h2h: h = h2h[h]
p2h[p] = h
h2ps = {}
for p, h in p2h.items() :
if h not in h2ps: h2ps[h] = []
if p not in h2ps[h]: h2ps[h].append(p )
d_half_1 = lgb.Dataset(X_half_1, label=y_half_1, categorical_feature=categoricals, free_raw_data=False)
 d_half_2 = lgb.Dataset(X_half_2, label=y_half_2, categorical_feature=categoricals, free_raw_data=False)
 watchlist_1 = [d_half_2, d_half_1]
watchlist_2 = [d_half_1, d_half_2]
params = {
"objective": "regression",
"boosting": "gbdt",
"num_leaves": 45,
"learning_rate": 0.02,
"feature_fraction": 0.9,
"reg_lambda": 2,
"metric": "rmse"
}<train_model>
def prefer(ps):
if len(ps)== 1: return ps[0]
best_p = ps[0]
best_s = p2size[best_p]
for i in range(1, len(ps)) :
p = ps[i]
s = p2size[p]
if s[0] * s[1] > best_s[0] * best_s[1]:
best_p = p
best_s = s
return best_p
h2p = {}
for h, ps in h2ps.items() :
h2p[h] = prefer(ps)
 len(h2p), list(h2p.items())[:5]
print("Building model with first half and validating on second half:")
 model_half_1 = lgb.train(params, train_set=d_half_1, num_boost_round=10, valid_sets=watchlist_1, verbose_eval=200, early_stopping_rounds=200)
 print("Building model with second half and validating on first half:")
 model_half_2 = lgb.train(params, train_set=d_half_2, num_boost_round=10, valid_sets=watchlist_2, verbose_eval=200, early_stopping_rounds=200 )<define_variables>
p2bb = pd.read_csv(BB_DF ).set_index("Image")
 old_stderr = sys.stderr
sys.stderr = open('/dev/null' if platform.system() != 'Windows' else 'nul', 'w')
 sys.stderr = old_stderr
img_shape =(384, 384, 1)
 anisotropy = 2.15
crop_margin = 0.05
models = [model_half_1,model_half_2]<drop_column>
def build_transform(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):

 rotation = np.deg2rad(rotation)
 shear = np.deg2rad(shear)
 rotation_matrix = np.array(
 [[np.cos(rotation), np.sin(rotation), 0], [-np.sin(rotation), np.cos(rotation), 0], [0, 0, 1]])
 shift_matrix = np.array([[1, 0, height_shift], [0, 1, width_shift], [0, 0, 1]])
 shear_matrix = np.array([[1, np.sin(shear), 0], [0, np.cos(shear), 0], [0, 0, 1]])
 zoom_matrix = np.array([[1.0 / height_zoom, 0, 0], [0, 1.0 / width_zoom, 0], [0, 0, 1]])
 shift_matrix = np.array([[1, 0, -height_shift], [0, 1, -width_shift], [0, 0, 1]])
 return np.dot(np.dot(rotation_matrix, shear_matrix), np.dot(zoom_matrix, shift_matrix))
del X_half_1,X_half_2,d_half_1,d_half_2,df_train, watchlist_1,watchlist_2,y_half_1,y_half_2
gc.collect()<prepare_x_and_y>
def read_cropped_image(p, augment):

 if p in h2p:
p = h2p[p]
size_x, size_y = p2size[p]
row = p2bb.loc[p]
x0, y0, x1, y1 = row['x0'], row['y0'], row['x1'], row['y1']
dx = x1 - x0
dy = y1 - y0
x0 -= dx * crop_margin
x1 += dx * crop_margin + 1
y0 -= dy * crop_margin
y1 += dy * crop_margin + 1
if x0 < 0:
x0 = 0
if x1 > size_x:
x1 = size_x
if y0 < 0:
y0 = 0
if y1 > size_y:
y1 = size_y
dx = x1 - x0
dy = y1 - y0
if dx > dy * anisotropy:
dy = 0.5 *(dx / anisotropy - dy)
 y0 -= dy
y1 += dy
else:
dx = 0.5 *(dy * anisotropy - dx)
 x0 -= dx
x1 += dx
trans = np.array([[1, 0, -0.5 * img_shape[0]], [0, 1, -0.5 * img_shape[1]], [0, 0, 1]])
 trans = np.dot(np.array([[(y1 - y0)/ img_shape[0], 0, 0], [0,(x1 - x0)/ img_shape[1], 0], [0, 0, 1]]), trans)
 if augment:
trans = np.dot(build_transform(
 random.uniform(-5, 5),
random.uniform(-5, 5),
random.uniform(0.8, 1.0),
random.uniform(0.8, 1.0),
random.uniform(-0.05 *(y1 - y0), 0.05 *(y1 - y0)) ,
random.uniform(-0.05 *(x1 - x0), 0.05 *(x1 - x0)) 
), trans)
 trans = np.dot(np.array([[1, 0, 0.5 *(y1 + y0)], [0, 1, 0.5 *(x1 + x0)], [0, 0, 1]]), trans)
 img = read_raw_image(p ).convert('L')
 img = img_to_array(img)
 matrix = trans[:2, :2]
offset = trans[:2, 2]
img = img.reshape(img.shape[:-1])
 img = affine_transform(img, matrix, offset, output_shape=img_shape[:-1], order=1, mode='constant',
cval=np.average(img))
img = img.reshape(img_shape)
 img -= np.mean(img, keepdims=True)
 img /= np.std(img, keepdims=True)+ K.epsilon()
return img
def read_for_training(p):

 return read_cropped_image(p, True)
 def read_for_validation(p):

 return read_cropped_image(p, False)
 p = list(tagged.keys())[312]
<load_from_csv>
def subblock(x, filter, **kwargs):
x = BatchNormalization()(x)
 y = x
y = Conv2D(filter,(1, 1), activation='relu', **kwargs )(y)
 y = BatchNormalization()(y)
 y = Conv2D(filter,(3, 3), activation='relu', **kwargs )(y)
 y = BatchNormalization()(y)
 y = Conv2D(K.int_shape(x)[-1],(1, 1), **kwargs )(y)
 y = Add()([x, y])
 y = Activation('relu' )(y)
 return y
def build_model(lr, l2, activation='sigmoid'):
regul = regularizers.l2(l2)
 optim = Adam(lr=lr)
 kwargs = {'padding': 'same', 'kernel_regularizer': regul}
inp = Input(shape=img_shape)
 x = Conv2D(64,(9, 9), strides=2, activation='relu', **kwargs )(inp)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 for _ in range(2):
x = BatchNormalization()(x)
 x = Conv2D(64,(3, 3), activation='relu', **kwargs )(x)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(128,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 64, **kwargs)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(256,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 64, **kwargs)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(384,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 96, **kwargs)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(512,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 128, **kwargs)
 x = GlobalMaxPooling2D()(x)
 branch_model = Model(inp, x)
 mid = 32
xa_inp = Input(shape=branch_model.output_shape[1:])
 xb_inp = Input(shape=branch_model.output_shape[1:])
 x1 = Lambda(lambda x: x[0] * x[1] )([xa_inp, xb_inp])
 x2 = Lambda(lambda x: x[0] + x[1] )([xa_inp, xb_inp])
 x3 = Lambda(lambda x: K.abs(x[0] - x[1]))([xa_inp, xb_inp])
 x4 = Lambda(lambda x: K.square(x))(x3)
 x = Concatenate()([x1, x2, x3, x4])
 x = Reshape(( 4, branch_model.output_shape[1], 1), name='reshape1' )(x)
 x = Conv2D(mid,(4, 1), activation='relu', padding='valid' )(x)
 x = Reshape(( branch_model.output_shape[1], mid, 1))(x)
 x = Conv2D(1,(1, mid), activation='linear', padding='valid' )(x)
 x = Flatten(name='flatten' )(x)
 x = Dense(1, use_bias=True, activation=activation, name='weighted-average' )(x)
 head_model = Model([xa_inp, xb_inp], x, name='head')
 img_a = Input(shape=img_shape)
 img_b = Input(shape=img_shape)
 xa = branch_model(img_a)
 xb = branch_model(img_b)
 x = head_model([xa, xb])
 model = Model([img_a, img_b], x)
 model.compile(optim, loss='binary_crossentropy', metrics=['binary_crossentropy', 'acc'])
 return model, branch_model, head_model
model, branch_model, head_model = build_model(64e-5, 0 )
df_test = pd.read_csv(f"{INPUT}test.csv")
 weather_test = pd.read_csv(f"{INPUT}weather_test.csv")
 df_test = reduce_mem_usage(df=df_test)
 weather_test = reduce_mem_usage(df=weather_test )<create_dataframe>
h2ws = {}
new_whale = 'new_whale'
for p, w in tagged.items() :
if w != new_whale:
h = p2h[p]
if h not in h2ws: h2ws[h] = []
if w not in h2ws[h]: h2ws[h].append(w)
 for h, ws in h2ws.items() :
if len(ws)> 1:
h2ws[h] = sorted(ws)
 w2hs = {}
for h, ws in h2ws.items() :
if len(ws)== 1:
w = ws[0]
if w not in w2hs: w2hs[w] = []
if h not in w2hs[w]: w2hs[w].append(h)
 for w, hs in w2hs.items() :
if len(hs)> 1:
w2hs[w] = sorted(hs )
df_test = prepare_data(df_test,bldg_metadata,weather_test,test=True,new_df=new_df )<feature_engineering>
train = []
for hs in w2hs.values() :
if len(hs)> 1:
train += hs
random.shuffle(train)
 train_set = set(train)
 w2ts = {}
for w, hs in w2hs.items() :
for h in hs:
if h in train_set:
if w not in w2ts:
w2ts[w] = []
if h not in w2ts[w]:
w2ts[w].append(h)
 for w, ts in w2ts.items() :
w2ts[w] = np.array(ts)
 t2i = {}
for i, t in enumerate(train):
t2i[t] = i
df_test = reduce_mem_usage(df_test )<drop_column>
class TrainingData(Sequence):
def __init__(self, score, steps=1000, batch_size=32):

 super(TrainingData, self ).__init__()
self.score = -score
self.steps = steps
self.batch_size = batch_size
for ts in w2ts.values() :
idxs = [t2i[t] for t in ts]
for i in idxs:
for j in idxs:
self.score[
i, j] = 10000.0
self.on_epoch_end()
def __getitem__(self, index):
start = self.batch_size * index
end = min(start + self.batch_size, len(self.match)+ len(self.unmatch))
size = end - start
assert size > 0
a = np.zeros(( size,)+ img_shape, dtype=K.floatx())
 b = np.zeros(( size,)+ img_shape, dtype=K.floatx())
 c = np.zeros(( size, 1), dtype=K.floatx())
 j = start // 2
for i in range(0, size, 2):
a[i, :, :, :] = read_for_training(self.match[j][0])
 b[i, :, :, :] = read_for_training(self.match[j][1])
 c[i, 0] = 1
a[i + 1, :, :, :] = read_for_training(self.unmatch[j][0])
 b[i + 1, :, :, :] = read_for_training(self.unmatch[j][1])
 c[i + 1, 0] = 0
j += 1
return [a, b], c
def on_epoch_end(self):
if self.steps <= 0: return
self.steps -= 1
self.match = []
self.unmatch = []
num_threads = 6
tmp = num_threads*[None]
threads = []
thread_input = num_threads*[None]
thread_idx = 0
batch = score.shape[0] //(num_threads-1)
 for start in range(0, score.shape[0], batch):
end = min(score.shape[0], start + batch)
 thread_input[thread_idx] = self.score[start:end, start:end]
thread_idx += 1
def worker(data_idx):
x,_,_ = lapjv(thread_input[data_idx])
 tmp[data_idx] = x
print("Start worker threads")
 for i in range(num_threads):
t = threading.Thread(target=worker, args=(i,), daemon=True)
 t.start()
threads.append(t)
 for t in threads:
if t is not None:
t.join()
x = np.concatenate(tmp)
 print("LAP completed")
 y = np.arange(len(x), dtype=np.int32)
 for ts in w2ts.values() :
d = ts.copy()
while True:
random.shuffle(d)
 if not np.any(ts == d): break
for ab in zip(ts, d): self.match.append(ab)
 for i, j in zip(x, y):
if i == j:
print(self.score)
 print(x)
 print(y)
 print(i, j)
 assert i != j
self.unmatch.append(( train[i], train[j]))
self.score[x, y] = 10000.0
self.score[y, x] = 10000.0
random.shuffle(self.match)
 random.shuffle(self.unmatch)
 assert len(self.match)== len(train)and len(self.unmatch)== len(train)
 def __len__(self):
return(len(self.match)+ len(self.unmatch)+ self.batch_size - 1)// self.batch_size
def __init__(self, score, steps=1000, batch_size=32):

 super(TrainingData, self ).__init__()
self.score = -score
self.steps = steps
self.batch_size = batch_size
for ts in w2ts.values() :
idxs = [t2i[t] for t in ts]
for i in idxs:
for j in idxs:
self.score[
i, j] = 10000.0
self.on_epoch_end()
def __getitem__(self, index):
start = self.batch_size * index
end = min(start + self.batch_size, len(self.match)+ len(self.unmatch))
size = end - start
assert size > 0
a = np.zeros(( size,)+ img_shape, dtype=K.floatx())
 b = np.zeros(( size,)+ img_shape, dtype=K.floatx())
 c = np.zeros(( size, 1), dtype=K.floatx())
 j = start // 2
for i in range(0, size, 2):
a[i, :, :, :] = read_for_training(self.match[j][0])
 b[i, :, :, :] = read_for_training(self.match[j][1])
 c[i, 0] = 1
a[i + 1, :, :, :] = read_for_training(self.unmatch[j][0])
 b[i + 1, :, :, :] = read_for_training(self.unmatch[j][1])
 c[i + 1, 0] = 0
j += 1
return [a, b], c
def on_epoch_end(self):
if self.steps <= 0: return
self.steps -= 1
self.match = []
self.unmatch = []
num_threads = 6
tmp = num_threads*[None]
threads = []
thread_input = num_threads*[None]
thread_idx = 0
batch = score.shape[0] //(num_threads-1)
 for start in range(0, score.shape[0], batch):
end = min(score.shape[0], start + batch)
 thread_input[thread_idx] = self.score[start:end, start:end]
thread_idx += 1
def worker(data_idx):
x,_,_ = lapjv(thread_input[data_idx])
 tmp[data_idx] = x
print("Start worker threads")
 for i in range(num_threads):
t = threading.Thread(target=worker, args=(i,), daemon=True)
 t.start()
threads.append(t)
 for t in threads:
if t is not None:
t.join()
x = np.concatenate(tmp)
 print("LAP completed")
 y = np.arange(len(x), dtype=np.int32)
 for ts in w2ts.values() :
d = ts.copy()
while True:
random.shuffle(d)
 if not np.any(ts == d): break
for ab in zip(ts, d): self.match.append(ab)
 for i, j in zip(x, y):
if i == j:
print(self.score)
 print(x)
 print(y)
 print(i, j)
 assert i != j
self.unmatch.append(( train[i], train[j]))
self.score[x, y] = 10000.0
self.score[y, x] = 10000.0
random.shuffle(self.match)
 random.shuffle(self.unmatch)
 assert len(self.match)== len(train)and len(self.unmatch)== len(train)
 def __len__(self):
return(len(self.match)+ len(self.unmatch)+ self.batch_size - 1)// self.batch_size
def __init__(self, score, steps=1000, batch_size=32):

 super(TrainingData, self ).__init__()
self.score = -score
self.steps = steps
self.batch_size = batch_size
for ts in w2ts.values() :
idxs = [t2i[t] for t in ts]
for i in idxs:
for j in idxs:
self.score[
i, j] = 10000.0
self.on_epoch_end()
def __getitem__(self, index):
start = self.batch_size * index
end = min(start + self.batch_size, len(self.match)+ len(self.unmatch))
size = end - start
assert size > 0
a = np.zeros(( size,)+ img_shape, dtype=K.floatx())
 b = np.zeros(( size,)+ img_shape, dtype=K.floatx())
 c = np.zeros(( size, 1), dtype=K.floatx())
 j = start // 2
for i in range(0, size, 2):
a[i, :, :, :] = read_for_training(self.match[j][0])
 b[i, :, :, :] = read_for_training(self.match[j][1])
 c[i, 0] = 1
a[i + 1, :, :, :] = read_for_training(self.unmatch[j][0])
 b[i + 1, :, :, :] = read_for_training(self.unmatch[j][1])
 c[i + 1, 0] = 0
j += 1
return [a, b], c
def on_epoch_end(self):
if self.steps <= 0: return
self.steps -= 1
self.match = []
self.unmatch = []
num_threads = 6
tmp = num_threads*[None]
threads = []
thread_input = num_threads*[None]
thread_idx = 0
batch = score.shape[0] //(num_threads-1)
 for start in range(0, score.shape[0], batch):
end = min(score.shape[0], start + batch)
 thread_input[thread_idx] = self.score[start:end, start:end]
thread_idx += 1
def worker(data_idx):
x,_,_ = lapjv(thread_input[data_idx])
 tmp[data_idx] = x
print("Start worker threads")
 for i in range(num_threads):
t = threading.Thread(target=worker, args=(i,), daemon=True)
 t.start()
threads.append(t)
 for t in threads:
if t is not None:
t.join()
x = np.concatenate(tmp)
 print("LAP completed")
 y = np.arange(len(x), dtype=np.int32)
 for ts in w2ts.values() :
d = ts.copy()
while True:
random.shuffle(d)
 if not np.any(ts == d): break
for ab in zip(ts, d): self.match.append(ab)
 for i, j in zip(x, y):
if i == j:
print(self.score)
 print(x)
 print(y)
 print(i, j)
 assert i != j
self.unmatch.append(( train[i], train[j]))
self.score[x, y] = 10000.0
self.score[y, x] = 10000.0
random.shuffle(self.match)
 random.shuffle(self.unmatch)
 assert len(self.match)== len(train)and len(self.unmatch)== len(train)
 def __len__(self):
return(len(self.match)+ len(self.unmatch)+ self.batch_size - 1)// self.batch_size
df_test = df_test.drop(columns=['start_ts','timestamp'])
<drop_column>
def set_lr(model, lr):
K.set_value(model.optimizer.lr, float(lr))
def get_lr(model):
return K.get_value(model.optimizer.lr)
 def score_reshape(score, x, y=None):

 if y is None:
m = np.zeros(( x.shape[0], x.shape[0]), dtype=K.floatx())
 m[np.triu_indices(x.shape[0], 1)] = score.squeeze()
m += m.transpose()
else:
m = np.zeros(( y.shape[0], x.shape[0]), dtype=K.floatx())
 iy, ix = np.indices(( y.shape[0], x.shape[0]))
ix = ix.reshape(( ix.size,))
iy = iy.reshape(( iy.size,))
m[iy, ix] = score.squeeze()
return m
def compute_score(verbose=1):

 features = branch_model.predict_generator(FeatureGen(train, verbose=verbose), max_queue_size=12, workers=6,
verbose=0)
 score = head_model.predict_generator(ScoreGen(features, verbose=verbose), max_queue_size=12, workers=6, verbose=0)
 score = score_reshape(score, features)
 return features, score
def make_steps(step, ampl):

 global w2ts, t2i, steps, features, score, histories
random.shuffle(train)
 w2ts = {}
for w, hs in w2hs.items() :
for h in hs:
if h in train_set:
if w not in w2ts: w2ts[w] = []
if h not in w2ts[w]: w2ts[w].append(h)
 for w, ts in w2ts.items() : w2ts[w] = np.array(ts)
 t2i = {}
for i, t in enumerate(train): t2i[t] = i
features, score = compute_score()
history = model.fit_generator(
 TrainingData(score + ampl * np.random.random_sample(size=score.shape), steps=step, batch_size=32),
initial_epoch=steps, epochs=steps + step, max_queue_size=12, workers=6, verbose=1 ).history
steps += step
del bldg_metadata,weather_test,new_df;gc.collect()<drop_column>
histories = []
steps = 0
tmp = keras.models.load_model('.. /input/reset-v3-100/siamese_v3_100')
 model.set_weights(tmp.get_weights())
 set_lr(model, 4e-5)
 make_steps(5, 0.25)
 set_lr(model, 4e-5)
 make_steps(5, 0.25)
 model.save('siamese_v3_110' )
test = df_test[feat_cols]<find_best_params>
set_lr(model, 4e-5)
 make_steps(5, 0.25)
 set_lr(model, 4e-5)
 make_steps(5, 0.25)
 model.save('siamese_v3_120' )
i=0
res=[]
step_size = 50000
for j in tqdm(range(int(np.ceil(test.shape[0]/50000)))) :
res.append(sum(np.expm1([model.predict(test.iloc[i:i+step_size])for model in models])/len(models)))
 i+=step_size<concatenate>
def prepare_submission(threshold, filename):

 vtop = 0
vhigh = 0
pos = [0, 0, 0, 0, 0, 0]
with open(filename, 'wt', newline='
')as f:
f.write('Image,Id
')
 for i, p in enumerate(tqdm(submit)) :
t = []
s = set()
a = score[i, :]
for j in list(reversed(np.argsort(a))):
h = known[j]
if a[j] < threshold and new_whale not in s:
pos[len(t)] += 1
s.add(new_whale)
 t.append(new_whale)
 if len(t)== 5: break;
for w in h2ws[h]:
assert w != new_whale
if w not in s:
if a[j] > 1.0:
vtop += 1
elif a[j] >= threshold:
vhigh += 1
s.add(w)
 t.append(w)
 if len(t)== 5: break;
if len(t)== 5: break;
if new_whale not in s: pos[5] += 1
assert len(t)== 5 and len(s)== 5
f.write(p + ',' + ' '.join(t[:5])+ '
')
 return vtop, vhigh, pos
<save_to_csv><EOS> 
tic = time.time()
h2ws = {}
for p, w in tagged.items() :
if w != new_whale:
h = p2h[p]
if h not in h2ws: h2ws[h] = []
if w not in h2ws[h]: h2ws[h].append(w)
 known = sorted(list(h2ws.keys()))
h2i = {}
for i, h in enumerate(known): h2i[h] = i
fknown = branch_model.predict_generator(FeatureGen(known), max_queue_size=20, workers=10, verbose=0)
 fsubmit = branch_model.predict_generator(FeatureGen(submit), max_queue_size=20, workers=10, verbose=0)
 score = head_model.predict_generator(ScoreGen(fknown, fsubmit), max_queue_size=20, workers=10, verbose=0)
 score = score_reshape(score, fknown, fsubmit)
 with open('score_matrix.pickle', 'wb')as fobj:
pickle.dump(score, fobj)
 prepare_submission(0.99, 'submission.csv')
 prepare_submission(0.97, 'submission_.csv' )