Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<import_modules>
!pip install tweet-preprocessor
!pip install lap
Lambda, MaxPooling2D, Reshape 
<load_from_csv>
import os
from google.cloud import storage, automl_v1beta1 as automl
import numpy as np
import pandas as pd
from sklearn import feature_extraction, linear_model, model_selection, preprocessing
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
import scipy as sp
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold, KFold, GridSearchCV
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score, confusion_matrix, f1_score
from sklearn.naive_bayes import GaussianNB, MultinomialNB
import seaborn as sns
import matplotlib.pyplot as plt
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
from nltk.stem import WordNetLemmatizer
import nltk
from collections import Counter
import string
import re
from nltk.corpus import wordnet as wn
from statistics import mean
import preprocessor
TRAIN_DF = '.. /input/humpback-whale-identification/train.csv'
SUB_Df = '.. /input/humpback-whale-identification/sample_submission.csv'
TRAIN = '.. /input/humpback-whale-identification/train/'
TEST = '.. /input/humpback-whale-identification/test/'
P2H = '.. /input/metadata/p2h.pickle'
P2SIZE = '.. /input/metadata/p2size.pickle'
BB_DF = ".. /input/metadata/bounding_boxes.csv"
tagged = dict([(p, w)for _, p, w in read_csv(TRAIN_DF ).to_records() ])
 submit = [p for _, p, _ in read_csv(SUB_Df ).to_records() ]
join = list(tagged.keys())+ submit<train_model>
from sklearn import metrics
if isfile(P2SIZE):
print("P2SIZE exists.")
 with open(P2SIZE, 'rb')as f:
p2size = pickle.load(f)
 else:
p2size = {}
for p in tqdm(join):
size = pil_image.open(expand_path(p)).size
p2size[p] = size<compute_test_metric>
stop = set(STOPWORDS ).union(set(['FAV' , 'RT']))
lemma = WordNetLemmatizer()
preprocessor.set_options(preprocessor.OPT.URL, preprocessor.OPT.MENTION, preprocessor.OPT.NUMBER, preprocessor.OPT.RESERVED)
 def clean(text):
text = preprocessor.clean(text)
 text = re.sub(r'[^\w\s]','',text)
 stop_free = " ".join([i for i in text.split(' ')if(i not in stop)])
 normalized = " ".join(lemma.lemmatize(word)for word in stop_free.split())
 return normalized
def match(h1, h2):
for p1 in h2ps[h1]:
for p2 in h2ps[h2]:
i1 = pil_image.open(expand_path(p1))
i2 = pil_image.open(expand_path(p2))
if i1.mode != i2.mode or i1.size != i2.size: return False
a1 = np.array(i1)
 a1 = a1 - a1.mean()
a1 = a1 / sqrt(( a1 ** 2 ).mean())
 a2 = np.array(i2)
 a2 = a2 - a2.mean()
a2 = a2 / sqrt(( a2 ** 2 ).mean())
 a =(( a1 - a2)** 2 ).mean()
if a > 0.1: return False
return True
if isfile(P2H):
print("P2H exists.")
 with open(P2H, 'rb')as f:
p2h = pickle.load(f)
 else:
p2h = {}
for p in tqdm(join):
img = pil_image.open(expand_path(p))
h = phash(img)
 p2h[p] = h
h2ps = {}
for p, h in p2h.items() :
if h not in h2ps: h2ps[h] = []
if p not in h2ps[h]: h2ps[h].append(p)
 hs = list(h2ps.keys())
 h2h = {}
for i, h1 in enumerate(tqdm(hs)) :
for h2 in hs[:i]:
if h1 - h2 <= 6 and match(h1, h2):
s1 = str(h1)
 s2 = str(h2)
 if s1 < s2: s1, s2 = s2, s1
h2h[s1] = s2
for p, h in p2h.items() :
h = str(h)
 if h in h2h: h = h2h[h]
p2h[p] = h
h2ps = {}
for p, h in p2h.items() :
if h not in h2ps: h2ps[h] = []
if p not in h2ps[h]: h2ps[h].append(p )<categorify>
train_df = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test_df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv" )
def prefer(ps):
if len(ps)== 1: return ps[0]
best_p = ps[0]
best_s = p2size[best_p]
for i in range(1, len(ps)) :
p = ps[i]
s = p2size[p]
if s[0] * s[1] > best_s[0] * best_s[1]:
best_p = p
best_s = s
return best_p
h2p = {}
for h, ps in h2ps.items() :
h2p[h] = prefer(ps)
 len(h2p), list(h2p.items())[:5]<set_options>
train_df.text = train_df.text.apply(clean)
 test_df.text = test_df.text.apply(clean )
p2bb = pd.read_csv(BB_DF ).set_index("Image")
 old_stderr = sys.stderr
sys.stderr = open('/dev/null' if platform.system() != 'Windows' else 'nul', 'w')
 sys.stderr = old_stderr
img_shape =(384, 384, 1)
 anisotropy = 2.15
crop_margin = 0.05<normalization>
PROJECT_ID = 'automl-kaggle-263107'
def build_transform(rotation, shear, height_zoom, width_zoom, height_shift, width_shift):

 rotation = np.deg2rad(rotation)
 shear = np.deg2rad(shear)
 rotation_matrix = np.array(
 [[np.cos(rotation), np.sin(rotation), 0], [-np.sin(rotation), np.cos(rotation), 0], [0, 0, 1]])
 shift_matrix = np.array([[1, 0, height_shift], [0, 1, width_shift], [0, 0, 1]])
 shear_matrix = np.array([[1, np.sin(shear), 0], [0, np.cos(shear), 0], [0, 0, 1]])
 zoom_matrix = np.array([[1.0 / height_zoom, 0, 0], [0, 1.0 / width_zoom, 0], [0, 0, 1]])
 shift_matrix = np.array([[1, 0, -height_shift], [0, 1, -width_shift], [0, 0, 1]])
 return np.dot(np.dot(rotation_matrix, shear_matrix), np.dot(zoom_matrix, shift_matrix))<prepare_x_and_y>
BUCKET_NAME = 'automl-disaster-tweet-cleaned'
BUCKET_REGION = 'us-central1'
def read_cropped_image(p, augment):

 if p in h2p:
p = h2p[p]
size_x, size_y = p2size[p]
row = p2bb.loc[p]
x0, y0, x1, y1 = row['x0'], row['y0'], row['x1'], row['y1']
dx = x1 - x0
dy = y1 - y0
x0 -= dx * crop_margin
x1 += dx * crop_margin + 1
y0 -= dy * crop_margin
y1 += dy * crop_margin + 1
if x0 < 0:
x0 = 0
if x1 > size_x:
x1 = size_x
if y0 < 0:
y0 = 0
if y1 > size_y:
y1 = size_y
dx = x1 - x0
dy = y1 - y0
if dx > dy * anisotropy:
dy = 0.5 *(dx / anisotropy - dy)
 y0 -= dy
y1 += dy
else:
dx = 0.5 *(dy * anisotropy - dx)
 x0 -= dx
x1 += dx
trans = np.array([[1, 0, -0.5 * img_shape[0]], [0, 1, -0.5 * img_shape[1]], [0, 0, 1]])
 trans = np.dot(np.array([[(y1 - y0)/ img_shape[0], 0, 0], [0,(x1 - x0)/ img_shape[1], 0], [0, 0, 1]]), trans)
 if augment:
trans = np.dot(build_transform(
 random.uniform(-5, 5),
random.uniform(-5, 5),
random.uniform(0.8, 1.0),
random.uniform(0.8, 1.0),
random.uniform(-0.05 *(y1 - y0), 0.05 *(y1 - y0)) ,
random.uniform(-0.05 *(x1 - x0), 0.05 *(x1 - x0)) 
), trans)
 trans = np.dot(np.array([[1, 0, 0.5 *(y1 + y0)], [0, 1, 0.5 *(x1 + x0)], [0, 0, 1]]), trans)
 img = read_raw_image(p ).convert('L')
 img = img_to_array(img)
 matrix = trans[:2, :2]
offset = trans[:2, 2]
img = img.reshape(img.shape[:-1])
 img = affine_transform(img, matrix, offset, output_shape=img_shape[:-1], order=1, mode='constant',
cval=np.average(img))
img = img.reshape(img_shape)
 img -= np.mean(img, keepdims=True)
 img /= np.std(img, keepdims=True)+ K.epsilon()
return img
def read_for_training(p):

 return read_cropped_image(p, True)
 def read_for_validation(p):

 return read_cropped_image(p, False)
 p = list(tagged.keys())[312]<categorify>
storage_client = storage.Client(project=PROJECT_ID)
 tables_gcs_client = automl.GcsClient(client=storage_client, bucket_name=BUCKET_NAME)
 automl_client = automl.AutoMlClient()
prediction_client = automl.PredictionServiceClient()
tables_client = automl.TablesClient(project=PROJECT_ID, region=BUCKET_REGION, client=automl_client, gcs_client=tables_gcs_client, prediction_client=prediction_client )
def subblock(x, filter, **kwargs):
x = BatchNormalization()(x)
 y = x
y = Conv2D(filter,(1, 1), activation='relu', **kwargs )(y)
 y = BatchNormalization()(y)
 y = Conv2D(filter,(3, 3), activation='relu', **kwargs )(y)
 y = BatchNormalization()(y)
 y = Conv2D(K.int_shape(x)[-1],(1, 1), **kwargs )(y)
 y = Add()([x, y])
 y = Activation('relu' )(y)
 return y
def build_model(lr, l2, activation='sigmoid'):
regul = regularizers.l2(l2)
 optim = Adam(lr=lr)
 kwargs = {'padding': 'same', 'kernel_regularizer': regul}
inp = Input(shape=img_shape)
 x = Conv2D(64,(9, 9), strides=2, activation='relu', **kwargs )(inp)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 for _ in range(2):
x = BatchNormalization()(x)
 x = Conv2D(64,(3, 3), activation='relu', **kwargs )(x)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(128,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 64, **kwargs)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(256,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 64, **kwargs)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(384,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 96, **kwargs)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(512,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = subblock(x, 128, **kwargs)
 x = GlobalMaxPooling2D()(x)
 branch_model = Model(inp, x)
 mid = 32
xa_inp = Input(shape=branch_model.output_shape[1:])
 xb_inp = Input(shape=branch_model.output_shape[1:])
 x1 = Lambda(lambda x: x[0] * x[1] )([xa_inp, xb_inp])
 x2 = Lambda(lambda x: x[0] + x[1] )([xa_inp, xb_inp])
 x3 = Lambda(lambda x: K.abs(x[0] - x[1]))([xa_inp, xb_inp])
 x4 = Lambda(lambda x: K.square(x))(x3)
 x = Concatenate()([x1, x2, x3, x4])
 x = Reshape(( 4, branch_model.output_shape[1], 1), name='reshape1' )(x)
 x = Conv2D(mid,(4, 1), activation='relu', padding='valid' )(x)
 x = Reshape(( branch_model.output_shape[1], mid, 1))(x)
 x = Conv2D(1,(1, mid), activation='linear', padding='valid' )(x)
 x = Flatten(name='flatten' )(x)
 x = Dense(1, use_bias=True, activation=activation, name='weighted-average' )(x)
 head_model = Model([xa_inp, xb_inp], x, name='head')
 img_a = Input(shape=img_shape)
 img_b = Input(shape=img_shape)
 xa = branch_model(img_a)
 xb = branch_model(img_b)
 x = head_model([xa, xb])
 model = Model([img_a, img_b], x)
 model.compile(optim, loss='binary_crossentropy', metrics=['binary_crossentropy', 'acc'])
 return model, branch_model, head_model
model, branch_model, head_model = build_model(64e-5, 0 )<feature_engineering>
bucket = storage.Bucket(storage_client, name=BUCKET_NAME)
 if not bucket.exists() :
bucket.create(location=BUCKET_REGION )
h2ws = {}
new_whale = 'new_whale'
for p, w in tagged.items() :
if w != new_whale:
h = p2h[p]
if h not in h2ws: h2ws[h] = []
if w not in h2ws[h]: h2ws[h].append(w)
 for h, ws in h2ws.items() :
if len(ws)> 1:
h2ws[h] = sorted(ws)
 w2hs = {}
for h, ws in h2ws.items() :
if len(ws)== 1:
w = ws[0]
if w not in w2hs: w2hs[w] = []
if h not in w2hs[w]: w2hs[w].append(h)
 for w, hs in w2hs.items() :
if len(hs)> 1:
w2hs[w] = sorted(hs )<categorify>
def upload_blob(bucket_name, source_file_name, destination_blob_name):

 bucket = storage_client.get_bucket(bucket_name)
 blob = bucket.blob(destination_blob_name)
 blob.upload_from_filename(source_file_name)
 print('File {} uploaded to {}.'.format(
 source_file_name,
destination_blob_name))
def download_to_kaggle(bucket_name,destination_directory,file_name,prefix=None):

 os.makedirs(destination_directory, exist_ok = True)
 full_file_path = os.path.join(destination_directory, file_name)
 blobs = storage_client.list_blobs(bucket_name,prefix=prefix)
 for blob in blobs:
blob.download_to_filename(full_file_path )
train = []
for hs in w2hs.values() :
if len(hs)> 1:
train += hs
random.shuffle(train)
 train_set = set(train)
 w2ts = {}
for w, hs in w2hs.items() :
for h in hs:
if h in train_set:
if w not in w2ts:
w2ts[w] = []
if h not in w2ts[w]:
w2ts[w].append(h)
 for w, ts in w2ts.items() :
w2ts[w] = np.array(ts)
 t2i = {}
for i, t in enumerate(train):
t2i[t] = i<define_variables>
train_df[['id','text','target']].to_csv('/kaggle/working/train.csv', index=False)
 test_df[['id','text']].to_csv('/kaggle/working/test.csv', index=False )
class TrainingData(Sequence):
def __init__(self, score, steps=1000, batch_size=32):

 super(TrainingData, self ).__init__()
self.score = -score
self.steps = steps
self.batch_size = batch_size
for ts in w2ts.values() :
idxs = [t2i[t] for t in ts]
for i in idxs:
for j in idxs:
self.score[
i, j] = 10000.0
self.on_epoch_end()
def __getitem__(self, index):
start = self.batch_size * index
end = min(start + self.batch_size, len(self.match)+ len(self.unmatch))
size = end - start
assert size > 0
a = np.zeros(( size,)+ img_shape, dtype=K.floatx())
 b = np.zeros(( size,)+ img_shape, dtype=K.floatx())
 c = np.zeros(( size, 1), dtype=K.floatx())
 j = start // 2
for i in range(0, size, 2):
a[i, :, :, :] = read_for_training(self.match[j][0])
 b[i, :, :, :] = read_for_training(self.match[j][1])
 c[i, 0] = 1
a[i + 1, :, :, :] = read_for_training(self.unmatch[j][0])
 b[i + 1, :, :, :] = read_for_training(self.unmatch[j][1])
 c[i + 1, 0] = 0
j += 1
return [a, b], c
def on_epoch_end(self):
if self.steps <= 0: return
self.steps -= 1
self.match = []
self.unmatch = []
_, _, x = lapjv(self.score)
 y = np.arange(len(x), dtype=np.int32)
 for ts in w2ts.values() :
d = ts.copy()
while True:
random.shuffle(d)
 if not np.any(ts == d): break
for ab in zip(ts, d): self.match.append(ab)
 for i, j in zip(x, y):
if i == j:
print(self.score)
 print(x)
 print(y)
 print(i, j)
 assert i != j
self.unmatch.append(( train[i], train[j]))
self.score[x, y] = 10000.0
self.score[y, x] = 10000.0
random.shuffle(self.match)
 random.shuffle(self.unmatch)
 assert len(self.match)== len(train)and len(self.unmatch)== len(train)
 def __len__(self):
return(len(self.match)+ len(self.unmatch)+ self.batch_size - 1)// self.batch_size
score = np.random.random_sample(size=(len(train), len(train)))
 data = TrainingData(score)
(a, b), c = data[0]<train_model>
upload_blob(BUCKET_NAME, '/kaggle/working/train.csv', 'train.csv')
 upload_blob(BUCKET_NAME, '/kaggle/working/test.csv', 'test.csv' )
def set_lr(model, lr):
K.set_value(model.optimizer.lr, float(lr))
def get_lr(model):
return K.get_value(model.optimizer.lr)
 def score_reshape(score, x, y=None):

 if y is None:
m = np.zeros(( x.shape[0], x.shape[0]), dtype=K.floatx())
 m[np.triu_indices(x.shape[0], 1)] = score.squeeze()
m += m.transpose()
else:
m = np.zeros(( y.shape[0], x.shape[0]), dtype=K.floatx())
 iy, ix = np.indices(( y.shape[0], x.shape[0]))
ix = ix.reshape(( ix.size,))
iy = iy.reshape(( iy.size,))
m[iy, ix] = score.squeeze()
return m
def compute_score(verbose=1):

 features = branch_model.predict_generator(FeatureGen(train, verbose=verbose), max_queue_size=12, workers=6,
verbose=0)
 score = head_model.predict_generator(ScoreGen(features, verbose=verbose), max_queue_size=12, workers=6, verbose=0)
 score = score_reshape(score, features)
 return features, score
def make_steps(step, ampl):

 global w2ts, t2i, steps, features, score, histories
random.shuffle(train)
 w2ts = {}
for w, hs in w2hs.items() :
for h in hs:
if h in train_set:
if w not in w2ts: w2ts[w] = []
if h not in w2ts[w]: w2ts[w].append(h)
 for w, ts in w2ts.items() : w2ts[w] = np.array(ts)
 t2i = {}
for i, t in enumerate(train): t2i[t] = i
features, score = compute_score()
history = model.fit_generator(
 TrainingData(score + ampl * np.random.random_sample(size=score.shape), steps=step, batch_size=32),
initial_epoch=steps, epochs=steps + step, max_queue_size=12, workers=6, verbose=1 ).history
steps += step
history['epochs'] = steps
history['ms'] = np.mean(score)
 history['lr'] = get_lr(model)
 print(history['epochs'], history['lr'], history['ms'])
 histories.append(history )<load_pretrained>
dataset_display_name = 'tweet_disaster_cleaned'
new_dataset = False
try:
dataset = tables_client.get_dataset(dataset_display_name=dataset_display_name)
 except:
new_dataset = True
dataset = tables_client.create_dataset(dataset_display_name )
def prepare_submission(threshold, filename):

 vtop = 0
vhigh = 0
pos = [0, 0, 0, 0, 0, 0]
with open(filename, 'wt', newline='
')as f:
f.write('Image,Id
')
 for i, p in enumerate(tqdm(submit)) :
t = []
s = set()
a = score[i, :]
for j in list(reversed(np.argsort(a))):
h = known[j]
if a[j] < threshold and new_whale not in s:
pos[len(t)] += 1
s.add(new_whale)
 t.append(new_whale)
 if len(t)== 5: break;
for w in h2ws[h]:
assert w != new_whale
if w not in s:
if a[j] > 1.0:
vtop += 1
elif a[j] >= threshold:
vhigh += 1
s.add(w)
 t.append(w)
 if len(t)== 5: break;
if len(t)== 5: break;
if new_whale not in s: pos[5] += 1
assert len(t)== 5 and len(s)== 5
f.write(p + ',' + ' '.join(t[:5])+ '
')
 return vtop, vhigh, pos<load_pretrained>
if new_dataset:
gcs_input_uris = ['gs://' + BUCKET_NAME + '/train.csv']
import_data_operation = tables_client.import_data(
 dataset=dataset,
gcs_input_uris=gcs_input_uris 
)
 print('Dataset
import_data_operation.result()
histories = []
steps = 0
if isfile('.. /input/piotte/mpiotte-standard.model'):
tmp = keras.models.load_model('.. /input/piotte/mpiotte-standard.model')
 model.set_weights(tmp.get_weights())
 tic = time.time()
h2ws = {}
for p, w in tagged.items() :
if w != new_whale:
h = p2h[p]
if h not in h2ws: h2ws[h] = []
if w not in h2ws[h]: h2ws[h].append(w)
 known = sorted(list(h2ws.keys()))
h2i = {}
for i, h in enumerate(known): h2i[h] = i
fknown1 = branch_model.predict_generator(FeatureGen(known), max_queue_size=20, workers=10, verbose=1)
 fsubmit1 = branch_model.predict_generator(FeatureGen(submit), max_queue_size=20, workers=10, verbose=1)
 score1 = head_model.predict_generator(ScoreGen(fknown1, fsubmit1), max_queue_size=20, workers=10, verbose=1)
 score1 = score_reshape(score1, fknown1, fsubmit1)
 tmp = keras.models.load_model('.. /input/piotte/mpiotte-bootstrap.model')
 model.set_weights(tmp.get_weights())
 tic = time.time()
h2ws = {}
for p, w in tagged.items() :
if w != new_whale:
h = p2h[p]
if h not in h2ws: h2ws[h] = []
if w not in h2ws[h]: h2ws[h].append(w)
 known = sorted(list(h2ws.keys()))
h2i = {}
for i, h in enumerate(known): h2i[h] = i
fknown2 = branch_model.predict_generator(FeatureGen(known), max_queue_size=20, workers=10, verbose=1)
 fsubmit2 = branch_model.predict_generator(FeatureGen(submit), max_queue_size=20, workers=10, verbose=1)
 score2 = head_model.predict_generator(ScoreGen(fknown2, fsubmit2), max_queue_size=20, workers=10, verbose=1)
 score2 = score_reshape(score2, fknown2, fsubmit2)
 else:
make_steps(10, 1000)
 ampl = 100.0
for _ in range(2):
print('noise ampl.= ', ampl)
 make_steps(5, ampl)
 ampl = max(1.0, 100 ** -0.1 * ampl)
<compute_test_metric>
ID_COLUMN = 'id'
score = 0.45*score1 + 0.55*score2<train_model>
TRAIN_BUDGET = 1000
model = None
model_display_name = 'tweet_disaster_model_clean'
try:
model = tables_client.get_model(model_display_name=model_display_name)
 except:
response = tables_client.create_model(
 model_display_name,
dataset=dataset,
train_budget_milli_node_hours=TRAIN_BUDGET,
exclude_column_spec_names=[TARGET_COLUMN,ID_COLUMN] 
)
 print('Create model operation: {}'.format(response.operation))
model = response.result()
print(model )
prepare_submission(0.92, 'submission_0.45_standard_0.55_boostrap.csv')
 toc = time.time()
print("Submission time: ",(toc - tic)/ 60.)<install_modules>
gcs_input_uris = 'gs://' + BUCKET_NAME + '/test.csv'
gcs_output_uri_prefix = 'gs://' + BUCKET_NAME + '/predictions'
batch_predict_response = tables_client.batch_predict(
 model=model,
gcs_input_uris=gcs_input_uris,
gcs_output_uri_prefix=gcs_output_uri_prefix, 
)
 print('Batch prediction operation: {}'.format(batch_predict_response.operation))
batch_predict_result = batch_predict_response.result()
batch_predict_response.metadata
!pip -q install aiohttp faiss-prebuilt pyxtools pymltools
!apt -qq install -y libopenblas-base libomp-dev
tensorflow.__version__<import_modules>
gcs_output_folder = batch_predict_response.metadata.batch_predict_details.output_info.gcs_output_directory.replace('gs://' + BUCKET_NAME + '/','')
 download_to_kaggle(BUCKET_NAME,'/kaggle/working','submissions.csv', prefix=gcs_output_folder )
MaxPooling2D
show_embedding, keras_convert_model_to_estimator_ckpt, InitFromPretrainedCheckpointHook, \
AbstractEstimator, estimator_iter_process, colab_save_file_func, OptimizerType, tf_model_fn, \
get_wsl_path, map_per_set, LossStepHookForTrain, ProcessMode, load_data_from_h5file, \
store_data_in_h5file, get_triplet_pair_np, map_per_image, show_distance_dense_plot, init_logger
FileCache, random_choice, NormType, get_pretty_float, image_utils
try:
except ImportError:
def apply_with_random_selector(x, func, num_cases):

 sel = tf.random_uniform([], maxval=num_cases, dtype=tf.int32)
 return control_flow_ops.merge([
func(control_flow_ops.switch(x, tf.equal(sel, case)) [1], case)
 for case in range(num_cases)])[0]
def distorted_bounding_box_crop(image,
bbox,
min_object_covered=0.1,
aspect_ratio_range=(0.75, 1.33),
area_range=(0.05, 1.0),
max_attempts=100,
scope=None):

 with tf.name_scope(scope, 'distorted_bounding_box_crop', [image, bbox]):
sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box(
 tf.shape(image),
bounding_boxes=bbox,
min_object_covered=min_object_covered,
aspect_ratio_range=aspect_ratio_range,
area_range=area_range,
max_attempts=max_attempts,
use_image_if_no_bounding_boxes=True)
 bbox_begin, bbox_size, distort_bbox = sample_distorted_bounding_box
cropped_image = tf.slice(image, bbox_begin, bbox_size)
 return cropped_image, distort_bbox
def _whale_gray_preprocess_for_train(image, height, width, bbox,
fast_mode=True,
scope=None,
add_image_summaries=True, mean_tf_func=None):

 with tf.name_scope(scope, 'distort_image', [image, height, width, bbox]):
if bbox is None:
bbox = tf.constant([0.0, 0.0, 1.0, 1.0],
dtype=tf.float32,
shape=[1, 1, 4])
 if image.shape[-1] == 3:
image = tf.image.rgb_to_grayscale(image)
 if image.dtype != tf.float32:
image = tf.image.convert_image_dtype(image, dtype=tf.float32)
 image_with_box = tf.image.draw_bounding_boxes(tf.expand_dims(image, 0), bbox)
 if add_image_summaries:
tf.summary.image('image_with_bounding_boxes', image_with_box)
 distorted_image, distorted_bbox = distorted_bounding_box_crop(
 image,
bbox,
min_object_covered=0.90,
aspect_ratio_range=(0.75, 1.33),
area_range=(0.90, 1.0),)
 distorted_image.set_shape([None, None, 1])
 image_with_distorted_box = tf.image.draw_bounding_boxes(
 tf.expand_dims(image, 0), distorted_bbox)
 if add_image_summaries:
tf.summary.image('images_with_distorted_bounding_box',
image_with_distorted_box)
 num_resize_cases = 1 if fast_mode else 4
distorted_image = apply_with_random_selector(
 distorted_image,
lambda x, method: tf.image.resize_images(x, [height, width], method),
num_cases=num_resize_cases)
 if add_image_summaries:
tf.summary.image('cropped_resized_image',
tf.expand_dims(distorted_image, 0))
distorted_image = tf.image.random_flip_left_right(distorted_image)
 distorted_image = tf.image.random_brightness(distorted_image, max_delta=30)
 if mean_tf_func is None:
mean_tf_func = tf_image_mean_inception
distorted_image = mean_tf_func(distorted_image)
 if add_image_summaries:
tf.summary.image('final_distorted_image',
tf.expand_dims(distorted_image, 0))
return distorted_image
def _whale_gray_preprocess_for_eval(image, height, width, bbox=None, scope=None, mean_tf_func=None):

 with tf.name_scope(scope, 'eval_image', [image, height, width]):
if image.shape[-1] == 3:
image = tf.image.rgb_to_grayscale(image)
 if image.dtype != tf.float32:
image = tf.image.convert_image_dtype(image, dtype=tf.float32)
 if bbox is not None:
image, distorted_bbox = distorted_bounding_box_crop(
 image, bbox,
min_object_covered=1.0,
aspect_ratio_range=(0.75, 1.33),
area_range=(0.99, 1.0), 
)
 image.set_shape([None, None, 1])
 if height and width:
image = tf.expand_dims(image, 0)
 image = tf.image.resize_bilinear(image, [height, width],
align_corners=False)
 image = tf.squeeze(image, [0])
 if mean_tf_func is None:
mean_tf_func = tf_image_mean_inception
image = mean_tf_func(image)
 return image
def _whale_rgb_preprocess_for_train(image, height, width, bbox, fast_mode=True, easy_train: bool = False,
scope=None, add_image_summaries=True, mean_tf_func=None):

 with tf.name_scope(scope, 'distort_image', [image, height, width, bbox]):
if bbox is None:
bbox = tf.constant([0.0, 0.0, 1.0, 1.0], dtype=tf.float32, shape=[1, 1, 4])
 if image.dtype != tf.float32:
image = tf.image.convert_image_dtype(image, dtype=tf.float32)
 image_with_box = tf.image.draw_bounding_boxes(tf.expand_dims(image, 0), bbox)
 if add_image_summaries:
tf.summary.image('image_with_bounding_boxes', image_with_box)
 if easy_train:
rotate_angle, min_cover = 0.01, 0.90
else:
rotate_angle, min_cover = 0.18, 0.30
image = tf.contrib.image.rotate(
 image,
angles=tf.random.uniform(shape=(1,), minval=-rotate_angle, maxval=rotate_angle,)[0],
interpolation="BILINEAR", 
)
 distorted_image, distorted_bbox = distorted_bounding_box_crop(
 image,
bbox,
min_object_covered=min_cover,
aspect_ratio_range=(0.75, 1.33),
area_range=(min_cover, 1.0),)
 distorted_image.set_shape([None, None, 3])
 image_with_distorted_box = tf.image.draw_bounding_boxes(
 tf.expand_dims(image, 0), distorted_bbox)
 if add_image_summaries:
tf.summary.image('images_with_distorted_bounding_box', image_with_distorted_box)
 num_resize_cases = 1 if fast_mode else 4
distorted_image = apply_with_random_selector(
 distorted_image,
lambda x, method: tf.image.resize_images(x, [height, width], method),
num_cases=num_resize_cases)
 if add_image_summaries:
tf.summary.image('cropped_resized_image', tf.expand_dims(distorted_image, 0))
distorted_image = tf.image.random_flip_left_right(distorted_image)
 distorted_image = tf.image.random_brightness(distorted_image, max_delta=30)
 if mean_tf_func is None:
mean_tf_func = tf_image_mean_inception
distorted_image = mean_tf_func(distorted_image)
 if add_image_summaries:
tf.summary.image('final_distorted_image', tf.expand_dims(distorted_image, 0))
return distorted_image
def _whale_rgb_preprocess_for_eval(image, height, width, bbox=None, scope=None, mean_tf_func=None):

 with tf.name_scope(scope, 'eval_image', [image, height, width]):
if image.dtype != tf.float32:
image = tf.image.convert_image_dtype(image, dtype=tf.float32)
 if bbox is not None:
image, distorted_bbox = distorted_bounding_box_crop(
 image, bbox,
min_object_covered=1.0,
aspect_ratio_range=(0.75, 1.33),
area_range=(0.99, 1.0), 
)
 image.set_shape([None, None, 3])
 if height and width:
image = tf.expand_dims(image, 0)
 image = tf.image.resize_bilinear(image, [height, width], align_corners=False)
 image = tf.squeeze(image, [0])
 if mean_tf_func is None:
mean_tf_func = tf_image_mean_inception
image = mean_tf_func(image)
 return image
def tf_image_mean_inception(image,):

 image = tf.subtract(image, 0.5)
 image = tf.multiply(image, 2.0)
 return image
def whale_siamese_image_mean_tf(image,):

 image = tf.subtract(image, tf.reduce_mean(image, keepdims=True))
dev_squared = tf.square(image - tf.reduce_mean(image, keepdims=True))
std_image = tf.sqrt(tf.reduce_mean(dev_squared, keepdims=True))
return tf.divide(image, std_image + K.epsilon())
 def whale_gray_preprocess_image(image, height, width,
is_training=False,
bbox=None,
fast_mode=True,
add_image_summaries=True, mean_tf_func=None):

 if is_training:
return _whale_gray_preprocess_for_train(
 image, height, width, bbox, fast_mode,
add_image_summaries=add_image_summaries, mean_tf_func=mean_tf_func)
 else:
return _whale_gray_preprocess_for_eval(image, height, width, bbox, mean_tf_func=mean_tf_func)
<load_from_csv>
preds_df = pd.read_csv("/kaggle/working/submissions.csv")
 preds_df = preds_df.sort_values(by=['id'])
 preds_df['target'] =(preds_df['target_1_score'] >= 0.5 ).astype(int )
def combine_csv(file_weight: dict, out_file: str):
sub_files = []
sub_weight = []
for csv_file, weight in file_weight.items() :
sub_files.append(csv_file)
 sub_weight.append(weight)
 place_weights = {}
for i in range(5):
place_weights[i] = 10 - i * 2
h_label = 'Image'
h_target = 'Id'
sub = [None] * len(sub_files)
 for i, file in enumerate(sub_files):
print("Reading {}: w={} - {}".format(i, sub_weight[i], file))
reader = csv.DictReader(open(file, "r"))
sub[i] = sorted(reader, key=lambda d: d[h_label])
 out = open(out_file, "w", newline='')
 writer = csv.writer(out)
 writer.writerow([h_label, h_target])
 p = 0
for row in sub[0]:
target_weight = {}
for s in range(len(sub_files)) :
row1 = sub[s][p]
for ind, trgt in enumerate(row1[h_target].split(' ')) :
target_weight[trgt] = target_weight.get(trgt, 0)+(place_weights[ind] * sub_weight[s])
 tops_trgt = sorted(target_weight, key=target_weight.get, reverse=True)[:5]
writer.writerow([row1[h_label], " ".join(tops_trgt)])
 p += 1
out.close()
def expend_bounding_box(boxes, pix: int = 5):
left, upper, right, lower = boxes
return max(0, left - pix), max(0, upper - pix), right + pix, lower + pix
class PathManager(object):
def __init__(self, mode: str = "default"):
self._mode = mode
self.init_all_path()
self.init_bounding_boxes()
self._submission_csv = None
@property
def data_set_root_path(self)-> str:
if self._mode == "default":
return get_wsl_path("E:/frkhit/Download/AI/data-set/kaggle/whale")
 elif self._mode == "fake":
return get_wsl_path("E:/frkhit/Download/AI/data-set/kaggle/whale/fake")
 elif self._mode == "colab":
return "/content"
elif self._mode == "kaggle":
return ".. /input/humpback-whale-identification"
return "./"
@property
def working_path(self):
if self._mode == "kaggle":
return "/kaggle/working/"
return self.data_set_root_path
@property
def output_path(self,):
if self._mode == "default" or self._mode == "fake":
return "/mnt/e/s"
return self.working_path
@property
def data_set_train_path(self):
return os.path.join(self.data_set_root_path, "train")
 @property
def data_set_test_path(self,):
return os.path.join(self.data_set_root_path, "test")
 @property
def train_csv(self,):
return os.path.join(self.data_set_root_path, "train.csv")
 @property
def submission_csv(self,):
if self._submission_csv:
return self._submission_csv
return os.path.join(self.output_path, "whale.submission.csv")
 @submission_csv.setter
def submission_csv(self, submission_csv):
self._submission_csv = submission_csv
@property
def bounding_boxes_csv(self,):
return os.path.join(self.working_path, "bounding_boxes.csv")
 @property
def ckpt_pretrained_rv250(self,):
if self._mode == "default" or self._mode == "fake":
return get_wsl_path(
 "E:/frkhit/Download/AI/pre-trained-model/resnet_v2_50.ckpt" 
)
 return os.path.join(self.working_path, "resnet_v2_50.ckpt")
 def init_bounding_boxes(self):
if not os.path.exists(self.bounding_boxes_csv):
download_big_file("https://raw.githubusercontent.com/frkhit/file_servers/master/bounding_boxes.csv",
self.bounding_boxes_csv)
 if not os.path.exists(self.bounding_boxes_csv):
raise ValueError("fail to download bounding_boxes_csv!")
 def init_image_net_model(self):
working_dir = os.path.dirname(self.ckpt_pretrained_mv2)
 if not os.path.exists(working_dir):
os.mkdir(working_dir)
 if not os.path.exists(self.ckpt_pretrained_mv2 + ".index"):
tar_file = "mobilenet_v2_1.0_224.tgz"
download_big_file("https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_1.0_224.tgz",
os.path.join(working_dir, tar_file))
if not os.path.exists(tar_file):
raise ValueError("fail to download pretrained model!")
 raw_dir = os.getcwd()
try:
os.chdir(working_dir)
 cmd = subprocess.Popen(["tar", "-xvf", tar_file])
 cmd.wait()
if not os.path.exists(self.ckpt_pretrained_mv2 + ".index"):
raise ValueError("fail to download pretrained model!")
 finally:
os.chdir(raw_dir)
 def download_resnet_v2_50(self):
working_dir = os.path.dirname(self.ckpt_pretrained_rv250)
 if not os.path.exists(working_dir):
os.mkdir(working_dir)
 if not os.path.exists(self.ckpt_pretrained_rv250):
tar_file = "resnet_v2_50_2017_04_14.tar.gz"
download_big_file("http://download.tensorflow.org/models/resnet_v2_50_2017_04_14.tar.gz",
os.path.join(working_dir, tar_file))
if not os.path.exists(tar_file):
raise ValueError("fail to download pretrained model!")
 raw_dir = os.getcwd()
try:
os.chdir(working_dir)
 cmd = subprocess.Popen(["tar", "-xzvf", tar_file])
 cmd.wait()
if not os.path.exists(self.ckpt_pretrained_rv250):
raise ValueError("fail to download pretrained model!")
 finally:
os.chdir(raw_dir)
 def init_all_path(self):
members = [attr for attr in dir(self)if not attr.startswith("__")]
for member in members:
if(member.endswith("path")or member.endswith("dir")) and not member.endswith("init_all_path"):
tmp_dir = getattr(self, member)
 if callable(tmp_dir):
tmp_dir = tmp_dir()
if not os.path.exists(tmp_dir):
try:
os.mkdir(tmp_dir)
 except Exception as e:
logging.error(e)
 @property
def is_kaggle(self):
return bool(self._mode == "kaggle")
 def create_fake_data(self, x: int = 1):
raw_path_manager = PathManager()
whales = pd.read_csv(raw_path_manager.train_csv)
 train_info = {}
for index, image_name in enumerate(whales.Image):
train_info[image_name] = whales.Id[index]
shutil.rmtree(self.data_set_root_path)
 os.mkdir(self.data_set_root_path)
 os.mkdir(self.data_set_test_path)
 test_image_list = list_files(raw_path_manager.data_set_test_path)
 for image_file in random.choices(test_image_list, k=10 * x):
shutil.copy(image_file, os.path.join(self.data_set_test_path, os.path.basename(image_file)))
 os.mkdir(self.data_set_train_path)
 raw_image_list = list_files(raw_path_manager.data_set_train_path)
 image_id_vs_image_file = {
os.path.basename(image_file): image_file for image_file in raw_image_list}
class_id_vs_image_list = {}
for image_id, class_id in train_info.items() :
class_id_vs_image_list.setdefault(class_id, [] ).append(image_id_vs_image_file[image_id])
 anchor_class_id_list = [class_id for class_id, _image_list in class_id_vs_image_list.items() if
len(_image_list)> 1]
all_anchor_class_id_list = list(train_info.values())
 try:
anchor_class_id_list.remove(WhaleDataUtils.blank_class_id)
 except Exception:
pass
to_move_list = []
for class_id in random.choices(anchor_class_id_list, k=2 * x):
to_move_list.extend(class_id_vs_image_list[class_id][:3 * x])
 for class_id in random.choices(all_anchor_class_id_list, k=3 * x):
to_move_list.extend(class_id_vs_image_list[class_id][:3 * x])
 to_move_list.extend(class_id_vs_image_list[WhaleDataUtils.blank_class_id][:4 * x])
 for image_file in to_move_list:
shutil.copy(image_file, os.path.join(self.data_set_train_path, os.path.basename(image_file)))
 with open(self.train_csv, "w")as f:
f.write("Image,Id
")
 for image_file in to_move_list:
image_id = os.path.basename(image_file)
 class_id = train_info[image_id]
f.write("{},{}
".format(image_id, class_id))
self.init_bounding_boxes()
class WhaleDataUtils(object):
blank_class_id = "new_whale"
not_gray_str = ".ng"
def __init__(self, path_manager: PathManager, loss_hook: LossStepHookForTrain = None,
gen_data_setting: dict = None):
self.logger = logging.getLogger(self.__class__.__name__)
 self._file = "./{}.pkl".format(self.__class__.__name__)
 self.path_manager = path_manager
self.loss_hook = loss_hook
self.file_cache = FileCache(pickle_file=self._file)
 self.dimension = 512
self.margin = 1.0
self.gen_data_setting = {
"x_train_num": 1,
"ignore_blank_prob": 0.95,
"ignore_single_prob": 0.2,
"stop_calc_feature": False,
"gen_data_by_random_prob": 0,
"use_norm_when_calc_apn": True,
}
if gen_data_setting:
self.gen_data_setting.update(gen_data_setting)
 _cache = None
if os.path.exists(self._file):
_cache = self.file_cache.get("cache")
 if not _cache:
self._cache =([], [], {}, {}, {}, [])
 else:
self._cache = _cache
self._baseline = None
self._to_use_random = False
def _update_cache(self):
self.file_cache.set("cache", self._cache)
 def clear_cache(self):
self.file_cache.cache.clear()
self.file_cache.set("__key__", "value", ttl=10)
 _pkl_file = self._file + ".feature.h5"
if os.path.exists(_pkl_file):
os.remove(_pkl_file)
 def load_train_info(self)-> dict:
if self._cache and self._cache[3]:
return self._cache[3]
whales = pd.read_csv(self.path_manager.train_csv)
 info = self._cache[3]
for index, image_name in enumerate(whales.Image):
info[image_name] = whales.Id[index]
self.logger.info("there are {} record in train csv!".format(len(info)))
 self._update_cache()
return info
def load_train_label_info(self)-> dict:
if self._cache and self._cache[4]:
return self._cache[4]
class_id_list = [class_id for class_id in self.load_train_info().values() ]
class_id_list = list(set(class_id_list))
class_id_list.sort()
info = self._cache[4]
for index, class_id in enumerate(class_id_list):
info[class_id] = index
self.logger.info("there are {} classes in train set!".format(len(info)))
 self._update_cache()
return info
def load_boxes_info(self):
if self._cache and self._cache[2]:
return self._cache[2]
boxes = pd.read_csv(self.path_manager.bounding_boxes_csv)
 info = self._cache[2]
for index, image_id in enumerate(boxes.Image):
info[image_id] =(boxes.x0[index], boxes.y0[index], boxes.x1[index], boxes.y1[index])
 self.logger.info("there are {} record in boxes csv!".format(len(info)))
 file_list = self.list_train_image_file().copy()
file_list.extend(self.list_test_image_file())
 for image_file in file_list:
base_name = os.path.basename(image_file)
 if base_name not in info:
raise ValueError("{} not in boxes info!".format(image_file))
self._update_cache()
return self._cache[2]
def get_boxes(self, image_file)->(int, int, int, int):
base_name = os.path.basename(image_file)
 return self.load_boxes_info() [base_name]
def list_train_image_file(self, ignore_blank: bool = False)-> list:
if not self._cache[0]:
image_list = list_files(self.path_manager.data_set_train_path)
 result_list = [file_name for file_name in image_list if file_name.endswith(".jpg")]
assert len(result_list)== len(image_list)
 self.logger.info("there are {} jpg in train path!".format(len(result_list)))
 self._cache[0].clear()
self._cache[0].extend(result_list)
 self._update_cache()
if not self._cache[5]:
for _image_file in self._cache[0]:
if self.get_class_id(_image_file)!= self.blank_class_id:
self._cache[5].append(_image_file)
 self._update_cache()
if ignore_blank is False:
return self._cache[0]
return self._cache[5]
def list_test_image_file(self)-> list:
if self._cache and self._cache[1]:
return self._cache[1]
image_list = list_files(self.path_manager.data_set_test_path)
 result_list = [file_name for file_name in image_list if file_name.endswith(".jpg")]
assert len(result_list)== len(image_list)
 self.logger.info("there are {} jpg in test path!".format(len(result_list)))
 self._cache[1].clear()
self._cache[1].extend(result_list)
 self._update_cache()
return self._cache[1]
def get_class_id(self, image_file: str)-> str:
base_name = os.path.basename(image_file)
 if base_name.endswith(self.not_gray_str):
base_name = base_name[:-len(self.not_gray_str)]
return self.load_train_info() [base_name]
def list_class_id_with_multi_instance(self)-> dict:
info = {}
for class_id in self.load_train_info().values() :
info.setdefault(class_id, 0)
 info[class_id] += 1
key_to_remove_list = []
for key, count in info.items() :
if count <= 1:
key_to_remove_list.append(key)
 for key in key_to_remove_list:
info.pop(key)
 return info
def get_label(self, image_file: str)-> int:
class_id = self.get_class_id(image_file)
 return self.load_train_label_info() [class_id]
def submit_test_result(self, test_image_list: list, result_list: list, feature_list: list = None):
if len(test_image_list)!= len(result_list):
self.logger.error("len(test_image_list)!= len(result_list)!")
 if feature_list and len(feature_list)!= len(test_image_list):
self.logger.error("len(test_image_list)!= len(feature_list)!")
 with open(self.path_manager.submission_csv, "w")as f:
f.write("Image,Id
")
 for index, image_file in enumerate(test_image_list):
f.write("{},{}
".format(os.path.basename(image_file),
" ".join([class_id for class_id in result_list[index]])))
 self.logger.info("success to save result in {}".format(self.path_manager.submission_csv))
if feature_list and feature_list[0]:
self.logger.info("saving feature list...")
 with open(self.path_manager.submission_csv + ".feature", "w")as f:
for index, image_file in enumerate(test_image_list):
f.write("{},{}
".format(os.path.basename(image_file), feature_list[index]))
self.logger.info("success to save feature result in {}".format(
 self.path_manager.submission_csv + ".feature"))
def get_random_baseline(self)->(float, float):

 if self._baseline is None:
class_count_dict = self.list_class_id_with_multi_instance()
class_list = sorted(class_count_dict.items() , key=lambda x: x[1], reverse=True)
 class_id_list = [_class_id for(_class_id, _count)in class_list]
random_class_id_list = [self.blank_class_id]
for _class_id in class_id_list:
if _class_id not in random_class_id_list:
random_class_id_list.append(_class_id)
 if len(random_class_id_list)>= 5:
break
labels = [self.get_class_id(image_file)for image_file in self.list_train_image_file() ]
class_id_result_list = [random_class_id_list] * len(labels)
 total_map_5 = map_per_set(labels=labels, predictions=class_id_result_list, k=5)
 total_map_1 = map_per_set(labels=labels, predictions=class_id_result_list, k=1)
 self._baseline =(total_map_5, total_map_1)
 return self._baseline
def list_debug_image_list(self, count: int = 128)-> list:
all_image_list, class_id_vs_image_list, all_class_id_list, anchor_class_id_list, single_class_id_list = \
self.get_basic_info(self.list_train_image_file())
 if count >= len(all_image_list):
return all_image_list
single_blank_class_files = []
anchor_class_id_set = set(anchor_class_id_list)
 for class_id, file_list in class_id_vs_image_list.items() :
if class_id not in anchor_class_id_set:
single_blank_class_files.extend(file_list)
 image_list = []
anchor_file_count = int(count *(1 - len(single_blank_class_files)/ len(all_image_list)))
 if(anchor_file_count + len(single_blank_class_files)) > len(all_image_list):
anchor_file_count = len(all_image_list)- len(single_blank_class_files)
 for class_id in anchor_class_id_list:
image_list.extend(class_id_vs_image_list[class_id])
 if len(image_list)>= anchor_file_count:
break
while len(image_list)> anchor_file_count:
image_list.pop(-1)
 image_list.extend(random_choice(single_blank_class_files, k=count - anchor_file_count, unique=True))
assert len(image_list)== count
return image_list
def get_basic_info(self, all_image_list: list = None)->(list, dict, list, list, list):

 if all_image_list is None:
all_image_list = list(self.list_train_image_file(ignore_blank=False))
image_id_vs_image_file = {os.path.basename(image_file): image_file for image_file in all_image_list}
class_id_vs_image_list = {}
for image_id, class_id in self.load_train_info().items() :
if image_id in image_id_vs_image_file:
class_id_vs_image_list.setdefault(class_id, [] ).append(image_id_vs_image_file[image_id])
 anchor_class_id_list = [class_id for class_id, _image_list in class_id_vs_image_list.items() if
len(_image_list)> 1]
all_class_id_list = list(class_id_vs_image_list.keys())
 single_class_id_list = list(set(all_class_id_list)- set(anchor_class_id_list))
try:
anchor_class_id_list.remove(self.blank_class_id)
 except Exception:
pass
try:
single_class_id_list.remove(self.blank_class_id)
 except Exception:
pass
return all_image_list, class_id_vs_image_list, all_class_id_list, anchor_class_id_list, single_class_id_list
def get_file_list(self, is_training: bool = False, shuffle: bool = False, num_epochs: int = 1,
batch_size: int = None, online_batch_count: int = 0)-> list:
def get_random_batch_list(batch_count: int, add_blank_class: bool, class_id_vs_image_list: dict,
anchor_class_id_list: list, all_image_list: list, single_class_id_list: list,
anchor_unique: bool, single_unique: bool,)-> list:
batch_list = []
if add_blank_class:
batch_list.append(random.choice(class_id_vs_image_list[self.blank_class_id]))
chosen_class_id_list = random_choice(anchor_class_id_list, k=batch_count, unique=anchor_unique)
 for class_id in chosen_class_id_list:
if len(class_id_vs_image_list[class_id])>= 2 * online_batch_count:
batch_list.extend(
 random_choice(class_id_vs_image_list[class_id], k=2 * online_batch_count, unique=True))
else:
batch_list.extend(class_id_vs_image_list[class_id])
 if(batch_count - len(batch_list)) < online_batch_count:
break
if len(batch_list)< batch_count:
if single_class_id_list:
batch_list.extend([
class_id_vs_image_list[class_id][0] for class_id in
random_choice(single_class_id_list, k=batch_count - len(batch_list),
unique=single_unique)])
 else:
batch_list.extend(
 random_choice(all_image_list, k=batch_count - len(batch_list), unique=anchor_unique))
else:
while len(batch_list)> batch_count:
batch_list.pop(-1)
 return batch_list
def create_file_list_by_random() -> list:
all_image_list, class_id_vs_image_list, all_class_id_list, anchor_class_id_list, single_class_id_list = \
self.get_basic_info()
total_count = int(num_epochs * len(all_image_list)* self.gen_data_setting["x_train_num"])
 batch_count = total_count // batch_size
multi_batch_list = []
_unique = True if len(anchor_class_id_list)>= batch_size else False
_unique_single = True if len(single_class_id_list)>= batch_size else False
_has_blank_class = True if class_id_vs_image_list.get(self.blank_class_id)else False
for i in range(batch_count):
batch_list = get_random_batch_list(
 batch_count=batch_size, add_blank_class=_has_blank_class,
class_id_vs_image_list=class_id_vs_image_list, anchor_class_id_list=anchor_class_id_list,
all_image_list=all_image_list, single_class_id_list=single_class_id_list,
anchor_unique=_unique, single_unique=_unique_single, 
)
 random.shuffle(batch_list)
 multi_batch_list.append(batch_list)
 return multi_batch_list
def create_file_list_by_distance() -> list:
feature_arr, file_arr = self._load_feature(create_if_not_exist=False)
 if feature_arr is None:
self.logger.warning(
 "no feature in h5, use create_file_list_by_random instead of create_file_list_by_distance!")
 return create_file_list_by_random()
if self.gen_data_setting.get("use_norm_when_calc_apn", False):
feature_arr, norm_float = NormType.all.normalize_and_return_norm(feature_arr)
 self.logger.info("norm of feature is {}<type: {}>".format(norm_float, type(norm_float)))
 assert isinstance(norm_float, float)
 margin = self.margin / norm_float
else:
margin = self.margin
_file_list = []
for i in range(file_arr.shape[0]):
_file_list.append(file_arr[i].decode("utf-8"))
all_image_list, class_id_vs_image_list, all_class_id_list, anchor_class_id_list, single_class_id_list = \
self.get_basic_info(_file_list)
 total_count = int(num_epochs * len(all_image_list)* self.gen_data_setting["x_train_num"])
 batch_count = total_count // batch_size
multi_batch_list = []
single_class_id_set = set(single_class_id_list)
 anchor_index_end = -1
_anchor_search_end = False
labels = np.zeros(shape=file_arr.shape, dtype=np.int)
 for i in range(file_arr.shape[0]):
_file_name = file_arr[i].decode("utf-8")
 labels[i] = self.get_label(_file_name)
 if not _anchor_search_end:
class_id = self.get_class_id(_file_name)
 if class_id in single_class_id_set:
_anchor_search_end = True
else:
anchor_index_end = i
anchor_feature = feature_arr[:anchor_index_end + 1]
_apn_np_start = time.time()
apn_list = get_triplet_pair_np(anchor_feature, all_feature=feature_arr, all_label=labels,
margin=margin, logger=self.logger)
 _apn_set = set()
for(a, p, n)in apn_list:
_apn_set.add(a)
 _apn_set.add(p)
 _apn_set.add(n)
 self.logger.info(
 "get {} hardest-apn-pairs in {} files, time cost {}s.Unique file in apn pairs is {}.".format(
 len(apn_list), feature_arr.shape[0], time.time() - _apn_np_start, len(_apn_set)))
 _unique = True if len(anchor_class_id_list)>= batch_size else False
_unique_single = True if len(single_class_id_list)>= batch_size else False
_has_blank_class = True if class_id_vs_image_list.get(self.blank_class_id)else False
_blank_file_id_set = set()
if _has_blank_class:
_blank_file_label = self.get_label(class_id_vs_image_list[self.blank_class_id][0])
 for i in range(labels.shape[0]):
if labels[i] == _blank_file_label:
_blank_file_id_set.add(i)
 assert len(_blank_file_id_set)== len(class_id_vs_image_list.get(self.blank_class_id, []))
if len(apn_list)<= batch_size or(len(_apn_set - _blank_file_id_set)+ 1)<= batch_size:
self.logger.warning("apn list is too small, use create_file_list_by_random instead!")
 return create_file_list_by_random()
tmp_apn_list = []
for i in range(batch_count):
batch_file_id_id = set()
_blank_exists = False
while True:
if len(tmp_apn_list)== 0:
tmp_apn_list.extend(list(apn_list))
random.shuffle(tmp_apn_list)
(a, p, n)= tmp_apn_list.pop(-1)
 for _file_id in [a, p, n]:
if _file_id in _blank_file_id_set:
if _blank_exists is False:
batch_file_id_id.add(_file_id)
 _blank_exists = True
else:
continue
else:
batch_file_id_id.add(_file_id)
 if len(batch_file_id_id)>= batch_size:
break
batch_list = [file_arr[file_id].decode("utf-8")for file_id in batch_file_id_id]
assert len(batch_list)>= batch_size
batch_list = batch_list[:batch_size]
random.shuffle(batch_list)
 multi_batch_list.append(batch_list)
 return multi_batch_list
if is_training:
if online_batch_count > 0:
self.logger.info("using create_file_list_by_distance...")
 if self._to_use_random:
_multi_batch_list = create_file_list_by_random()
self._to_use_random = False
else:
_multi_batch_list = create_file_list_by_distance()
if shuffle is True:
random.shuffle(_multi_batch_list)
 image_file_list = []
for _batch_list in _multi_batch_list:
image_file_list.extend(_batch_list)
 return image_file_list
else:
return self.list_train_image_file()
return self.list_test_image_file()
def _list_file_to_calc_feature(self)-> list:
if random.random() < self.gen_data_setting["ignore_blank_prob"]:
all_image_list = list(self.list_train_image_file(ignore_blank=True))
if random.random() < self.gen_data_setting["ignore_single_prob"]:
_, class_id_vs_image_list, _, anchor_class_id_list, _ = \
self.get_basic_info(all_image_list)
 all_file_list = []
anchor_class_id_set = set(anchor_class_id_list)
 for class_id, _file_list in class_id_vs_image_list.items() :
if class_id in anchor_class_id_set:
all_file_list.extend(_file_list)
 self.logger.info("list train image file [ignore_single, ignore_blank] to calc feature!")
 return all_file_list
else:
self.logger.info("list train image file [ignore_blank] to calc feature!")
 return all_image_list
self.logger.info("list all train image file to calc feature!")
 return list(self.list_train_image_file())
 def calc_feature(self, callback, epoch_num: int = None):

 if self.gen_data_setting.get("stop_calc_feature", False):
self.logger.info("no need to calc feature: stop_calc_feature is True!")
 return
if epoch_num is not None and epoch_num == 0 and os.path.exists(self._file + ".feature.h5"):
self.logger.info("no need to calc feature: feature file exist and epoch_num == 0!")
 return
if random.random() < self.gen_data_setting.get("gen_data_by_random_prob", 0):
self.logger.info("no need to calc feature: going to gen data by random!")
 self._to_use_random = True
return
file_list = self._list_file_to_calc_feature()
if not file_list:
self.logger.info("no need to calc feature!")
 return
self.logger.info("calculating feature of {} file...".format(len(file_list)))
 feature_list, feature_file_list = callback(file_list, mode=ProcessMode.train)
 self.logger.info("success to calculate feature of {} file!".format(len(feature_file_list)))
 self.update_feature(feature_list, feature_file_list, is_training=True)
 def update_feature(self, feature_list, file_list, is_training: bool):

 if not is_training:
self.logger.info("give up update-feature for no train mode!")
 return
if not feature_list:
return
self._save_feature(feature_list, file_list)
 def _save_feature(self, feature_list: list, file_list: list):
_pkl_file = self._file + ".feature.h5"
if os.path.exists(_pkl_file):
os.remove(_pkl_file)
 all_image_list, class_id_vs_image_list, all_class_id_list, anchor_class_id_list, single_class_id_list = \
self.get_basic_info(all_image_list=file_list)
 all_file_list = []
for class_id in anchor_class_id_list:
all_file_list.extend(class_id_vs_image_list[class_id])
 for class_id in single_class_id_list:
all_file_list.extend(class_id_vs_image_list[class_id])
 all_file_list.extend(class_id_vs_image_list.get(self.blank_class_id, []))
assert len(all_image_list)== len(all_file_list)
 sort_index_list = []
_dict = {}
for i, file_name in enumerate(file_list):
_dict[file_name] = i
for file_name in all_file_list:
sort_index_list.append(_dict[file_name])
 file_arr = np.asarray([f.encode("utf-8")for f in all_file_list], dtype=np.string_)
 feature_arr = np.zeros(shape=(len(all_file_list), self.dimension), dtype=np.float32)
 for index, key_index in enumerate(sort_index_list):
feature_arr[index][:] = feature_list[key_index].reshape(( self.dimension,))
_pkl_file = self._file + ".feature.h5"
store_data_in_h5file(_pkl_file, [feature_arr, file_arr], key_list=["feature", "file"])
 def _load_feature(self, create_if_not_exist: bool = True)->(np.ndarray, np.ndarray):

 _pkl_file = self._file + ".feature.h5"
if os.path.exists(_pkl_file):
feature, file_arr = None, None
try:
feature, file_arr = load_data_from_h5file(_pkl_file, key_list=["feature", "file"])
 except Exception as e:
os.remove(_pkl_file)
 self.logger.error(e)
 if feature is not None:
self.logger.info("feature shape is {}, file_arr shape is {}".format(feature.shape, file_arr.shape))
assert feature.shape[1] == self.dimension
assert file_arr.shape[0] == feature.shape[0]
return feature, file_arr
if not create_if_not_exist:
return None, None
all_image_list, class_id_vs_image_list, all_class_id_list, anchor_class_id_list, single_class_id_list = \
self.get_basic_info()
all_file_list = []
for class_id in anchor_class_id_list:
all_file_list.extend(class_id_vs_image_list[class_id])
 for class_id in single_class_id_list:
all_file_list.extend(class_id_vs_image_list[class_id])
 all_file_list.extend(class_id_vs_image_list[self.blank_class_id])
 assert len(all_image_list)== len(all_file_list)
 file_arr = np.asarray([f.encode("utf-8")for f in all_file_list], dtype=np.string_)
 feature = np.zeros(shape=(len(all_file_list), self.dimension))
return feature, file_arr 
<import_modules>
preds_df[['id','target']].to_csv("submission.csv", index=False )
class WhaleRankingUtils(object):
def __init__(self, data_utils: WhaleDataUtils, top_k: int = 5):
self.logger = logging.getLogger(self.__class__.__name__)
 self.top_k = top_k
self.data_utils = data_utils
def simple_rank(self, result_list: list, distance_cutoff: float = None, only_distance_fit: bool = False)-> list:

 _class_id_list = []
_class_id_set = set()
if distance_cutoff is None:
for result in result_list:
class_id = result[1]
if class_id not in _class_id_set:
_class_id_list.append(class_id)
 _class_id_set.add(class_id)
 if len(_class_id_list)< self.top_k:
if self.data_utils.blank_class_id not in _class_id_set:
_class_id_list.append(self.data_utils.blank_class_id)
 _class_id_set.add(self.data_utils.blank_class_id)
 else:
if result_list[0][0] > distance_cutoff:
_class_id_list.append(self.data_utils.blank_class_id)
 if only_distance_fit:
return _class_id_list
else:
_class_id_list.append(result_list[0][1])
 _class_id_set.add(_class_id_list[0])
 for result in result_list[1:]:
if result[0] <= distance_cutoff:
class_id = result[1]
if class_id not in _class_id_set:
_class_id_list.append(class_id)
 _class_id_set.add(class_id)
 else:
if only_distance_fit:
return _class_id_list
else:
break
if len(_class_id_list)< self.top_k:
if self.data_utils.blank_class_id not in _class_id_set:
_class_id_list.append(self.data_utils.blank_class_id)
 _class_id_set.add(self.data_utils.blank_class_id)
 if len(_class_id_list)< self.top_k:
for result in result_list[1:]:
if result[0] > distance_cutoff:
class_id = result[1]
if class_id not in _class_id_set:
_class_id_list.append(class_id)
 _class_id_set.add(class_id)
 return _class_id_list
def recreate_class_id_list(self, cutoff: float, class_id_list: list, feature_list: list)-> list:
new_class_id_list = []
count = 0
for i, class_list in enumerate(class_id_list):
_new_class_list = list(class_list)
 if feature_list[i][0][0] <= cutoff or _new_class_list[0] == self.data_utils.blank_class_id:
new_class_id_list.append(_new_class_list)
 continue
try:
found_index = _new_class_list.index(self.data_utils.blank_class_id)
 if found_index > -1:
_new_class_list.pop(found_index)
 except ValueError:
pass
_new_class_list.insert(0, self.data_utils.blank_class_id)
 new_class_id_list.append(_new_class_list)
 count += 1
self.logger.info("recreate_class_id_list change {}/{} record".format(count, len(class_id_list)))
 return new_class_id_list
def get_similar_cutoff(self, validating_labels: list, validating_feature_list: list)-> float:
_distance_list = []
_, class_id_vs_image_list, _, anchor_class_id_list, _ = \
self.data_utils.get_basic_info(self.data_utils.list_train_image_file())
 anchor_class_set = set(anchor_class_id_list)
 for i, class_id in enumerate(validating_labels):
if class_id not in anchor_class_set:
continue
for(_distance, _class_id, _file_name)in validating_feature_list[i]:
if class_id == _class_id:
_distance_list.append(_distance)
 self.logger.info("distance list: len is {}, first 5 is {}".format(len(_distance_list), _distance_list[:5]))
distance_mean, distance_std = show_distance_dense_plot(
 np.asarray(_distance_list), self.data_utils.path_manager.submission_csv + ".distance.jpg")
 distance_similar_cutoff = distance_mean + 3 * distance_std
self.logger.info("distance_similar_cutoff is {}".format(distance_similar_cutoff))
return distance_similar_cutoff
@staticmethod
def get_feature_list(feature_file: str)->(list, list):
feature_list = []
file_id_list = []
with open(feature_file, "r", encoding="utf-8")as f:
for line in f:
if len(line)< 2:
continue
line = line.rstrip()
file_id_list.append(line.split(",")[0])
 feature_list.append(eval(line[line.find(",")+ 1:]))
return file_id_list, feature_list
def analyze_validating_result(self, feature_file: str):
file_id_list, feature_list = self.get_feature_list(feature_file)
 class_id_result_list = [self.simple_rank(feature)[:self.top_k] for feature in feature_list]
labels = self.list_class_id(file_id_list)
 total_map_5 = map_per_set(labels=labels, predictions=class_id_result_list, k=5)
 total_map_1 = map_per_set(labels=labels, predictions=class_id_result_list, k=1)
 self.logger.info("validating data result is: map@5 is {}, map@1 is {}!".format(total_map_5, total_map_1))
distance_similar_cutoff = self.get_similar_cutoff(labels, feature_list)
 for i, image_id in enumerate(file_id_list):
map_5 = map_per_image(label=labels[i], predictions=class_id_result_list[i], k=5)
 if map_5 < 1.0:
self.logger.info("map@5 is {}
True: {}, Predict with cutoff is {}
{}
".format(
 map_5, labels[i],
self.data_utils.blank_class_id if feature_list[i][0][0] > distance_similar_cutoff else
feature_list[i][0][1],
feature_list[i])
)
 new_class_id_result = self.recreate_class_id_list(
 cutoff=distance_similar_cutoff, class_id_list=class_id_result_list, feature_list=feature_list)
 total_map_5 = map_per_set(labels=labels, predictions=new_class_id_result, k=5)
 total_map_1 = map_per_set(labels=labels, predictions=new_class_id_result, k=1)
 self.logger.info(
 "validating data result with cutoff is: map@5 is {}, map@1 is {}!".format(total_map_5, total_map_1))
def compare_submission_file(self, file_1, file_2):
def _compare_same_count(_result_1, _result_2, info):
assert set(_result_1.keys())== set(_result_2.keys())
 all_same_count = 0
for image_id, result_list in _result_1.items() :
if ",".join(result_list[:self.top_k])== ",".join(_result_2[image_id][:self.top_k]):
all_same_count += 1
self.logger.info("[{}] same count is {}/{}".format(info, all_same_count, len(_result_1)))
 def _compare_top1_count(_result_1, _result_2, info):
assert set(_result_1.keys())== set(_result_2.keys())
 all_same_count = 0
for image_id, result_list in _result_1.items() :
if result_list[0] == _result_2[image_id][0]:
all_same_count += 1
self.logger.info("[{}] same top1 count is {}/{}".format(info, all_same_count, len(_result_1)))
 def _compare_map(_result_1, _result_2, info):
assert set(_result_1.keys())== set(_result_2.keys())
 _file_id_list = list(_result_1.keys())
 _file_id_list.sort()
labels = self.list_class_id(_file_id_list)
 _class_id_result_list_1 = [_result_1[_image_id] for _image_id in _file_id_list]
map5_1 = map_per_set(labels=labels, predictions=_class_id_result_list_1, k=5)
 map1_1 = map_per_set(labels=labels, predictions=_class_id_result_list_1, k=1)
 _class_id_result_list_2 = [_result_2[_image_id] for _image_id in _file_id_list]
map5_2 = map_per_set(labels=labels, predictions=_class_id_result_list_2, k=5)
 map1_2 = map_per_set(labels=labels, predictions=_class_id_result_list_2, k=1)
 self.logger.info("[{}] map@5 is {}/{}, map@1 is {}/{}".format(
 info,
get_pretty_float(map5_1, count=3),
get_pretty_float(map5_2, count=3),
get_pretty_float(map1_1, count=3),
get_pretty_float(map1_2, count=3), 
))
result_1, result_2 = self.read_submission_file(file_1), self.read_submission_file(file_2)
 result_1_x, result_2_x = self._get_result_from_feature_file(file_1), self._get_result_from_feature_file(file_2)
 result_1_true, result_2_true = self._get_result_by_cutoff(file_1), self._get_result_by_cutoff(file_2)
 self.logger.info("Start to compare:
")
 self.logger.info("compare map5, map1")
 _compare_map(result_1, result_2, "map, raw, resenet vs siamese")
 _compare_map(result_1, result_1_x, "map, raw, resenet, csv vs feature")
 _compare_map(result_2, result_2_x, "map, raw, siamese, csv vs feature")
 _compare_map(result_1_true, result_2_true, "map, true, resenet vs siamese")
 _compare_map(result_1, result_1_true, "map, resent, raw vs true")
 _compare_map(result_2, result_2_true, "map, siamese, raw vs true")
 self.logger.info("End
")
 self.logger.info("compare top5")
 _compare_same_count(result_1, result_2, "top5, raw, resenet vs siamese")
 _compare_same_count(result_1_true, result_2_true, "top5, true, resenet vs siamese")
 _compare_same_count(result_1, result_1_true, "top5, resent, raw vs true")
 _compare_same_count(result_2, result_2_true, "top5, siamese, raw vs true")
 self.logger.info("End
")
 self.logger.info("compare top1")
 _compare_top1_count(result_1, result_2, "top1, raw, resenet vs siamese")
 _compare_top1_count(result_1_true, result_2_true, "top1, true, resenet vs siamese")
 _compare_top1_count(result_1, result_1_true, "top1, resent, raw vs true")
 _compare_top1_count(result_2, result_2_true, "top1, siamese, raw vs true")
 self.logger.info("End
")
 @staticmethod
def read_submission_file(csv_file: str)-> dict:
submission = pd.read_csv(csv_file)
 _result = {}
for index, _image_id in enumerate(submission.Image):
_result[_image_id] = submission.Id[index].rstrip().split(" ")
 return _result
def _get_result_from_feature_file(self, csv_file: str)-> dict:
feature_file = csv_file + ".feature"
assert os.path.exists(feature_file)
 cutoff, _, file_id_list, feature_list = self._read_feature_file(feature_file)
 _new_result = {}
for _index, file_id in enumerate(file_id_list):
_new_result[file_id] = self.simple_rank(
 feature_list[_index], distance_cutoff=cutoff, only_distance_fit=False)
 return _new_result
def _get_result_by_cutoff(self, csv_file: str)-> dict:
feature_file = csv_file + ".feature"
assert os.path.exists(feature_file)
 cutoff, labels, file_id_list, feature_list = self._read_feature_file(feature_file)
 self.logger.info("cutoff is {}".format(get_pretty_float(cutoff, count=3)))
 _new_result = {}
for _index, file_id in enumerate(file_id_list):
_new_result[file_id] = self.simple_rank(
 feature_list[_index], distance_cutoff=cutoff, only_distance_fit=True)
 return _new_result
def _read_feature_file(self, feature_file):
assert os.path.exists(feature_file)
 file_id_list, feature_list = self.get_feature_list(feature_file)
 labels = self.list_class_id(file_id_list)
 cutoff = self.get_similar_cutoff(labels, feature_list)
 return cutoff, labels, file_id_list, feature_list
@staticmethod
def _calc_weight_score(feature_list, file_id_list, cutoff)-> dict:
keep_count = 10
cutoff_score = 5.0
_new_result = {}
for _index, file_id in enumerate(file_id_list):
_new_result[file_id] = {}
_sort_num = -1
for(distance, class_id, file_name)in feature_list[_index]:
if class_id not in _new_result[file_id]:
_sort_num += 1
_new_result[file_id][class_id] = keep_count - _sort_num
if distance < cutoff:
_new_result[file_id][class_id] += cutoff_score
if _sort_num >= keep_count:
break
return _new_result
@staticmethod
def _get_result_by_weight_score(score_dict: dict, top_k: int = 5)-> dict:
def _sort_by_value(info_dict: dict)-> list:
_tuple = sorted(info_dict.items() , key=lambda x: x[1], reverse=True)
 return [_class_id for(_class_id, _weight)in _tuple]
result_dict = {}
for image_id, score in score_dict.items() :
result_dict[image_id] = _sort_by_value(score)[:top_k]
return result_dict
def combine_submission_file(self, validating_file_list: list, submission_file_list: list, output_file: str):
def _create_final_result(_result_dict: dict):
_file_list = []
_result_list = []
for _image_id, _class_list in _result_dict.items() :
_file_list.append(_image_id)
 _result_list.append(_class_list)
 return _file_list, _result_list
def _update_weight_score(all_dict: dict, one_dict: dict, model_weight: float):
for _image_id, _score_dict in one_dict.items() :
if _image_id not in all_dict:
all_dict[_image_id] = {}
for _class_id, _score in _score_dict.items() :
all_dict[_image_id].setdefault(_class_id, 0.0)
 all_dict[_image_id][_class_id] += _score * model_weight
return all_dict
def eval_result(_result_dict: dict):
_file_list, _result_list = _create_final_result(_result_dict)
 _labels = self.list_class_id(_file_list)
 _map5 = map_per_set(labels=_labels, predictions=_result_list, k=5)
 _map1 = map_per_set(labels=_labels, predictions=_result_list, k=1)
 return _map5, _map1
assert len(validating_file_list)== len(submission_file_list)
 for i, validating_submission_csv in enumerate(validating_file_list):
predict_submission_csv = submission_file_list[i]
assert ".".join(validating_submission_csv.split(".")[:3])== ".".join(predict_submission_csv.split(".")[:3])
 assert os.path.exists(validating_submission_csv + ".feature")
 assert os.path.exists(predict_submission_csv + ".feature")
 total_predict_score_dict = {}
total_validate_score_dict = {}
total_eval_result_list = []
for i, validating_submission_csv in enumerate(validating_file_list):
predict_submission_csv = submission_file_list[i]
cutoff, _, v_file_id_list, v_feature_list = self._read_feature_file(validating_submission_csv + ".feature")
 v_model_weight = float("0.{}".format(validating_submission_csv.split(".")[-3]))
validating_score_dict = self._calc_weight_score(
 feature_list=v_feature_list, file_id_list=v_file_id_list, cutoff=cutoff)
 total_validate_score_dict = _update_weight_score(
 total_validate_score_dict, validating_score_dict, model_weight=v_model_weight)
 p_file_id_list, p_feature_list = self.get_feature_list(predict_submission_csv + ".feature")
 predict_score_dict = self._calc_weight_score(
 feature_list=p_feature_list, file_id_list=p_file_id_list, cutoff=cutoff)
 total_predict_score_dict = _update_weight_score(
 total_predict_score_dict, predict_score_dict, model_weight=v_model_weight)
 _validate_result_dict = self.read_submission_file(validating_submission_csv)
 total_eval_result_list.append(eval_result(_validate_result_dict))
_validate_result_dict = self._get_result_by_weight_score(total_validate_score_dict, top_k=self.top_k)
 total_eval_result_list.append(eval_result(_validate_result_dict))
self.logger.info("after combine {} models:
validate map@5 is {}/{},
validate map@1 is {}/{}
".format(
 len(validating_file_list),
total_eval_result_list[-1][0], [x[0] for x in total_eval_result_list[:-1]],
total_eval_result_list[-1][1], [x[1] for x in total_eval_result_list[:-1]], 
))
raw_submission = self.data_utils.path_manager.submission_csv
try:
predict_result_dict = self._get_result_by_weight_score(total_predict_score_dict, top_k=self.top_k)
 predict_file_list, predict_result_list = _create_final_result(predict_result_dict)
 self.data_utils.path_manager.submission_csv = output_file
self.data_utils.submit_test_result(test_image_list=predict_file_list, result_list=predict_result_list)
 self.logger.info("save final submission result in {}".format(self.data_utils.path_manager.submission_csv))
finally:
self.data_utils.path_manager.submission_csv = raw_submission
def debug_map_csv(self, csv_file):
self.logger.info("debug map set from submission file...
 ")
 result_dict = self.read_submission_file(csv_file)
 class_id_result_list = []
file_id_list = []
for _image_id, _result_list in result_dict.items() :
file_id_list.append(_image_id)
 class_id_result_list.append(_result_list)
 labels = self.list_class_id(file_id_list)
 self.logger.info("map5 is {}".format(map_per_set(labels=labels, predictions=class_id_result_list, k=5)))
 self.logger.info("map1 is {}".format(map_per_set(labels=labels, predictions=class_id_result_list, k=1)))
 def list_class_id(self, file_id_list)-> list:
_, class_id_vs_image_list, _, _, _ = self.data_utils.get_basic_info(self.data_utils.list_train_image_file())
 class_id_vs_image_count = {
class_id: len(_image_list)for class_id, _image_list in class_id_vs_image_list.items() }
return [self.get_class_id(class_id_vs_image_count, _image_id)for _image_id in file_id_list]
def get_class_id(self, class_id_vs_image_count, image_file_or_id)-> str:
_class_id = self.data_utils.get_class_id(image_file_or_id)
 if class_id_vs_image_count[_class_id] > 1:
return _class_id
else:
return self.data_utils.blank_class_id
class SimpleSearch(object):
model = None
def __init__(self, data_utils: WhaleDataUtils, shape: tuple, dimension: int, top_k: int = 5,
norm_type: NormType = NormType.none, more_search_top_k: int = None):
self.logger = logging.getLogger(self.__class__.__name__)
 self.data_utils = data_utils
self.faiss_index_path = os.path.join(self.data_utils.path_manager.working_path, self.__class__.__name__)
 self.shape = shape
self.dimension = dimension
self.top_k = top_k
self._distance_cutoff = None
self.manager = ImageIndexUtils(
 index_dir=self.faiss_index_path,
dimension=self.dimension)
 self._norm_type = norm_type
if more_search_top_k is None or more_search_top_k < 0:
more_search_top_k = max(20, self.top_k)
 self._search_top_k = more_search_top_k + self.top_k
def show_embedding(self, mode: ProcessMode):
key_name = "train" if mode == ProcessMode.train else "test"
list_image_file = self.data_utils.list_train_image_file if mode == ProcessMode.train \
else self.data_utils.list_test_image_file
log_dir = os.path.join(self.faiss_index_path, "logs_{}".format(key_name))
self.logger.info("trying to show {} embedding...".format(key_name))
feature_list, image_list = self.calc_or_load_feature(mode)
 self.logger.info("got {}-image feature: {}/{}".format(
 key_name, len(image_list), len(list_image_file())) 
)
 labels = []
if mode == ProcessMode.train:
for image_file in image_list:
labels.append(self.data_utils.get_class_id(image_file))
else:
for image_file in image_list:
labels.append(os.path.basename(image_file))
show_embedding(feature_list=feature_list, labels=labels, log_dir=log_dir)
 self.logger.info("success to show {} embedding!".format(key_name))
def list_features(self, mode: ProcessMode)->(list, list):

 raise NotImplementedError
def calc_or_load_feature(self, mode: ProcessMode)->(list, list):
key_name = "train" if mode == ProcessMode.train else "test"
feature_npy = os.path.join(self.faiss_index_path, "{}.npy.pkl".format(key_name))
self.logger.info("{}ing image...".format(key_name))
if not os.path.exists(feature_npy):
self.logger.info("calculating feature of {} image...".format(key_name))
feature_list, image_list = self.list_features(mode)
 self.logger.info("success to calculate feature of {} image!".format(key_name))
with open(feature_npy, "wb")as f:
pickle.dump(( feature_list, image_list), f)
 else:
self.logger.info("loading feature of {} image...".format(key_name))
with open(feature_npy, "rb")as f:
feature_list, image_list = pickle.load(f)
 self.logger.info("success to load feature of {} image!".format(key_name))
return feature_list, image_list
def train(self):
if os.path.exists(self.manager.manager.faiss_index_file):
self.logger.info("train before, no need to train again!")
 return
feature_list, image_list = self.calc_or_load_feature(ProcessMode.train)
 self.logger.info("got train-image feature: {}/{}".format(
 len(image_list), len(self.data_utils.list_train_image_file())))
 image_info_list = []
for index, image_file in enumerate(image_list):
image_info_list.append({
"index": index,
"file": image_file,
"class_id": self.data_utils.get_class_id(image_file)} 
)
 self.manager.add_images(feature_list, image_info_list=image_info_list)
 self.logger.info("success to train image!")
 def test(self):
feature_list, image_list = self.calc_or_load_feature(ProcessMode.test)
 if len(image_list)!= len(self.data_utils.list_test_image_file()):
self.logger.warning("got test-image file: {}/{}".format(
 len(image_list), len(self.data_utils.list_test_image_file())))
 assert len(image_list)% len(self.data_utils.list_test_image_file())== 0
self.logger.info("got test-image feature: {}/{}".format(
 len(image_list), len(self.data_utils.list_test_image_file())))
 num_feature = len(image_list)// len(self.data_utils.list_test_image_file())
 self.logger.info("testing image...")
 test_image_list = [image_list[index * num_feature] for index in
range(len(self.data_utils.list_test_image_file())) ]
class_id_result_list, test_feature_list = self.search(
 image_list=test_image_list, feature_list=feature_list, return_feature_list=True)
 self.logger.info("success to test image!")
 assert len(class_id_result_list)== len(self.data_utils.list_test_image_file())
 self.data_utils.submit_test_result(test_image_list=test_image_list,
result_list=class_id_result_list,
feature_list=test_feature_list)
 if self._distance_cutoff is None:
self.logger.info("submit predict result[without distance cutoff] in {}".format(
 self.data_utils.path_manager.submission_csv))
else:
self.logger.info("submit predict result[with distance cutoff {}] in {}".format(
 get_pretty_float(self._distance_cutoff, count=3), self.data_utils.path_manager.submission_csv))
def search(self, image_list: list, feature_list: list, cache_file: str = None, ignore_same: bool = False,
return_feature_list: bool = True)->(list, list):
if cache_file is None:
cache_file = os.path.join(self.faiss_index_path, "cache.pkl")
 cache = FileCache(cache_file)
 assert len(feature_list)% len(image_list)== 0
num_feature = len(feature_list)// len(image_list)
 if num_feature > 1:
self.logger.info("1 image vs {} feature when searching!".format(num_feature))
src_list = []
for index in range(len(image_list)) :
image_file = image_list[index]
if not cache.get(image_file):
src_list.append(( image_file, feature_list[index * num_feature:(index + 1)* num_feature]))
whale_rank_utils = WhaleRankingUtils(data_utils=self.data_utils, top_k=self.top_k)
 extend = False
_show_log_per_steps = 100 if len(image_list)< 1000 else 1000
for part_src_list in iter_list_with_size(src_list, size=10240 // num_feature):
feature_list = []
for(_, _feature_list)in part_src_list:
feature_list.extend(_feature_list)
 raw_result_iterator = self.manager.image_search_iterator(
 feature_list=feature_list,
top_k=self._search_top_k,
extend=extend 
)
 index = -1
cache_list = []
for raw_image_result_list in raw_result_iterator:
index += 1
if index % num_feature == 0:
_image_file = part_src_list[index // num_feature][0] if ignore_same else None
image_result_list = []
for(faiss_image_index, faiss_image_extend_list, faiss_distance)in raw_image_result_list:
image_result = self.manager.get_image_info(faiss_image_index)
 if _image_file and image_result.get("file")and image_result.get("file" ).find(_image_file)== 0:
continue
image_result_list.append(( faiss_distance, image_result["class_id"], image_result["file"]))
if(index + 1)% num_feature == 0:
image_result_list = sorted(image_result_list, key=lambda x: x[0], reverse=False)
 _class_id_list = whale_rank_utils.simple_rank(image_result_list,
distance_cutoff=self._distance_cutoff)
 if len(_class_id_list)< self.top_k:
self.logger.debug("length of class_id_list < {}!".format(self.top_k))
if return_feature_list:
cache_list.append(( _class_id_list[:self.top_k], list(image_result_list)))
 else:
cache_list.append(( _class_id_list[:self.top_k], None))
if index % _show_log_per_steps == 0:
self.logger.info("got class_id_list of {} images".format(index))
for j, _class_id_list_feature_list in enumerate(cache_list):
cache.unsafe_set(part_src_list[j][0], _class_id_list_feature_list)
 cache.set(part_src_list[0][0], cache_list[0])
 class_id_result_list = []
search_feature_list = []
for image_file in image_list:
_class_id_list, _search_feature_list = cache.get(image_file)
 class_id_result_list.append(_class_id_list)
 search_feature_list.append(_search_feature_list)
 self.logger.info("success to parse search result for test images!")
 return class_id_result_list, search_feature_list
def _calc_map(self, info, data_percent, labels, class_id_result_list,
image_list, multi_index_list)->(float, float):
fake_map5 = map_per_set(labels=labels, predictions=class_id_result_list, k=5)
 fake_map1 = map_per_set(labels=labels, predictions=class_id_result_list, k=1)
 multi_map_5 = map_per_set(
 labels=[labels[index] for index in multi_index_list],
predictions=[class_id_result_list[index] for index in multi_index_list],
k=5)
 multi_map_1 = map_per_set(
 labels=[labels[index] for index in multi_index_list],
predictions=[class_id_result_list[index] for index in multi_index_list],
k=1)
 new_index_list = []
for _index, _image_file in enumerate(image_list):
if self.data_utils.get_class_id(_image_file)== self.data_utils.blank_class_id:
new_index_list.append(_index)
 new_map_5 = map_per_set(
 labels=[labels[index] for index in new_index_list],
predictions=[class_id_result_list[index] for index in new_index_list],
k=5)
 new_map_1 = map_per_set(
 labels=[labels[index] for index in new_index_list],
predictions=[class_id_result_list[index] for index in new_index_list],
k=1)
 self.logger.info(
 "map@5 for validating data[{} of train data, {}] is: standard {}, new-whale {}, all {}!".format(
 data_percent, info, multi_map_5, new_map_5, fake_map5)
)
 self.logger.info(
 "map@1 for validating data[{} of train data, {}] is: standard {}, new-whale {}, all {}!".format(
 data_percent, info, multi_map_1, new_map_1, fake_map1)
)
 return multi_map_5, multi_map_1
def validate(self, data_percent: float = 1.0)->(float, float):

 train_feature_list, train_image_list = self.calc_or_load_feature(ProcessMode.train)
 num_feature = len(train_image_list)// len(self.data_utils.list_train_image_file())
 if len(train_image_list)!= len(self.data_utils.list_train_image_file()):
self.logger.warning("got validating-image file: {}/{}".format(
 len(train_image_list), len(self.data_utils.list_train_image_file())))
 assert len(train_image_list)% len(self.data_utils.list_train_image_file())== 0
chosen_index_list = random_choice([index for index in range(len(self.data_utils.list_train_image_file())) ],
k=int(len(self.data_utils.list_train_image_file())* data_percent),
unique=True)
 if num_feature == 1:
image_list = [train_image_list[index] for index in chosen_index_list]
feature_list = [train_feature_list[index] for index in chosen_index_list]
else:
image_list = [train_image_list[index * num_feature] for index in chosen_index_list]
feature_list = []
for index in chosen_index_list:
feature_list.extend(train_feature_list[index * num_feature:(index + 1)* num_feature])
 self.logger.info("validating image...")
 _raw_class_id_result_list, validating_feature_list = self.search(
 image_list=image_list, feature_list=feature_list, ignore_same=True, return_feature_list=True)
 self.logger.info("success to validate image!")
 image_id_vs_image_file = {os.path.basename(image_file): image_file for image_file in image_list}
class_id_vs_image_list = {}
for image_id, class_id in self.data_utils.load_train_info().items() :
class_id_vs_image_list.setdefault(class_id, [] ).append(image_id_vs_image_file[image_id])
 class_id_vs_image_count = {
class_id: len(_image_list)for class_id, _image_list in class_id_vs_image_list.items() }
rank_utils = WhaleRankingUtils(data_utils=self.data_utils, top_k=self.top_k)
 labels = [rank_utils.get_class_id(class_id_vs_image_count, image_file)for image_file in image_list]
multi_index_list = []
for _index, _image_file in enumerate(image_list):
if class_id_vs_image_count[self.data_utils.get_class_id(_image_file)] > 1:
multi_index_list.append(_index)
 distance_cutoff = rank_utils.get_similar_cutoff(
 validating_labels=labels, validating_feature_list=validating_feature_list)
 self.logger.info("got distance_cutoff {} and use it!".format(get_pretty_float(distance_cutoff, count=3)))
 self._distance_cutoff = distance_cutoff
class_id_result_list = rank_utils.recreate_class_id_list(
 cutoff=self._distance_cutoff, class_id_list=_raw_class_id_result_list, feature_list=validating_feature_list)
 total_map_5, total_map_1 = self._calc_map(
 "with distance cutoff {}".format(get_pretty_float(self._distance_cutoff, count=3)) ,
data_percent, labels, class_id_result_list, image_list, multi_index_list 
)
 random_map5, random_map1 = self.data_utils.get_random_baseline()
self.logger.info("random baseline for validating data[{} of train data]: map@5 is {}, map@1 is {}!".format(
 data_percent, random_map5, random_map1))
raw_submission_csv = self.data_utils.path_manager.submission_csv
try:
self.data_utils.path_manager.submission_csv = \
raw_submission_csv + ".{}.validate.csv".format(str(( total_map_5 + total_map_1)/ 2)[2:])
 self.data_utils.submit_test_result(
 test_image_list=image_list, result_list=class_id_result_list, feature_list=validating_feature_list)
 self.logger.info("submit validating result[{} of train data, with distance cutoff {}] in {}".format(
 data_percent, get_pretty_float(distance_cutoff, count=3), self.data_utils.path_manager.submission_csv))
finally:
self.data_utils.path_manager.submission_csv = raw_submission_csv
return total_map_5, total_map_1
class TripletSearch(SimpleSearch):

 model = None
def __init__(self, instance, data_utils: WhaleDataUtils, clear_cache_after_exists: bool,
norm_type: NormType = NormType.l2):
self.instance = instance
super(TripletSearch, self ).__init__(
 data_utils, shape=(None,), dimension=instance.params.dimension, top_k=5)
 self.smart_iterator = None
self.feature_record = {}
self._clear_cache_after_exists = clear_cache_after_exists
self._norm_type = norm_type
def list_features(self, mode: ProcessMode)->(list, list):

 if "train" not in self.feature_record and "test" not in self.feature_record:
train_feature_list, train_feature_file_list = self.instance.list_features(mode=ProcessMode.train)
 test_feature_list, test_feature_file_list = self.instance.list_features(mode=ProcessMode.test)
 feature = np.vstack(train_feature_list + test_feature_list ).\
reshape(( len(train_feature_list)+ len(test_feature_list), self.dimension))
if self._norm_type is None:
normalized_feature = feature
else:
self.logger.info("using {} to normalize feature".format(self._norm_type))
normalized_feature = self._norm_type.normalize(feature)
 self.feature_record["train"] = train_feature_file_list
self.feature_record["test"] = test_feature_file_list
self.feature_record["feature"] = normalized_feature
if mode == ProcessMode.train:
feature_file_list = self.feature_record["train"]
feature_list = [f.reshape(1, self.dimension)for f in
self.feature_record["feature"][:len(feature_file_list)]]
else:
feature_file_list = self.feature_record["test"]
feature_list = [f.reshape(1, self.dimension)for f in
self.feature_record["feature"][len(self.feature_record["train"]):]]
assert len(feature_list)== len(feature_file_list)
 assert feature_list[0].shape ==(1, self.dimension)
 return feature_list, feature_file_list
def clear_cache(self,):
if os.path.exists(self.faiss_index_path):
try:
if not self.data_utils.path_manager.is_kaggle:
tmp_dir = os.path.join(os.path.dirname(self.faiss_index_path), self.instance.__class__.__name__)
 if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
 shutil.copytree(self.faiss_index_path, tmp_dir)
 self.logger.info("backup Search Cache in {}".format(os.path.abspath(tmp_dir)))
 shutil.rmtree(self.faiss_index_path)
 except Exception as e:
self.logger.error(e, exc_info=True)
 def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if self._clear_cache_after_exists:
self.clear_cache() 
<choose_model_class>
tfidf_vectorizer = feature_extraction.text.TfidfVectorizer(ngram_range =(1,2), stop_words='english',strip_accents='unicode' )
class TripletLossModelCNN(AbstractEstimator):
def __init__(self, train_ckpt_dir, data_utils: WhaleDataUtils, timeout: int = int(3600 * 5),
pretrained_ckpt_file: str = None):
super(TripletLossModelCNN, self ).__init__(
 model_name="TripletLoss",
train_ckpt_dir=train_ckpt_dir,
pretrained_ckpt_file=pretrained_ckpt_file 
)
 self.data_utils = data_utils
self.timeout = timeout + time.time()
self.batch_size = 8
self._features_images_key = "images"
self._features_filename_key = "filenames"
self._features_embedding_key = "embeddings"
self.params = Params({
"num_channels": 1,
"margin": 1.0,
"dimension": 512,
"image_size": 384,
"online_batch_count": 4,
})
 self.learning_rate = 64e-5
self.image_tmp_size = 400
assert self.image_tmp_size >= self.params.image_size
self.train_loss_hook = LossStepHookForTrain(log_after_run=True, log_after_end=False,
show_log_per_steps=10, run_after_run_per_steps=5)
 self.data_utils.loss_hook = self.train_loss_hook
self.using_training_status_of_train_data = None
self.optimizer_type = OptimizerType.adam
def show_embedding(self, mode: ProcessMode, count: int = -1, remove_log_dir_if_exists: bool = False,
norm_type: NormType = NormType.all):
key_name = "train" if mode == ProcessMode.train else "test"
log_dir = os.path.join(self.TRAIN_CKPT_DIR, "logs_{}".format(key_name))
if os.path.exists(log_dir)and remove_log_dir_if_exists:
shutil.rmtree(log_dir)
 self.logger.info("trying to show {} embedding...".format(key_name))
if count > 1:
if mode == ProcessMode.train:
image_list = self.data_utils.list_debug_image_list(count=count)
 else:
image_list = self.data_utils.get_file_list(
 online_batch_count=0, is_training=bool(mode == ProcessMode.train))
image_list = image_list[:count]
if len(image_list)!= count:
self.logger.warning("only have {} files".format(len(image_list)))
 else:
image_list = self.data_utils.get_file_list(
 online_batch_count=0, is_training=bool(mode == ProcessMode.train))
if mode == ProcessMode.train:
label_list = [self.data_utils.get_class_id(image_file)for image_file in image_list]
else:
label_list = [os.path.basename(image_file)for image_file in image_list]
feature_list, _ = self.list_features(file_list=image_list)
 if norm_type is not None:
feature = np.vstack(feature_list ).reshape(( len(feature_list), self.params.dimension))
self.logger.info("using {} to normalize feature".format(norm_type))
normalized_feature = norm_type.normalize(feature)
 feature_list = [f.reshape(1, self.params.dimension)for f in normalized_feature]
self.logger.info("got {}-image feature: {}/{}".format(
 key_name, len(feature_list), len(image_list)) 
)
 show_embedding(feature_list=feature_list, labels=label_list, log_dir=log_dir)
 self.logger.info("success to show {} embedding!".format(key_name))
def show_predict_result(self, count: int = 10, top_k: int = 5):
tmp_feature_list = glob.glob(
 os.path.join(self.data_utils.path_manager.output_path, "whale.sub_.*.submission.csv.feature"))
tmp_feature_list = sorted(tmp_feature_list, key=lambda x: os.path.getmtime(x), reverse=True)
 file_id_list, feature_list = WhaleRankingUtils.get_feature_list(feature_file=tmp_feature_list[0])
 if count < len(file_id_list):
index_list = random_choice(src_list=[i for i in range(len(file_id_list)) ], k=count, unique=True)
 else:
index_list = [i for i in range(len(file_id_list)) ]
image_file_list = []
for index in index_list:
_image_file_list = [os.path.join(self.data_utils.path_manager.data_set_test_path, file_id_list[index])]
_count = 0
for(_distance, _class_id, _file_name)in feature_list[index]:
_image_file_list.append(
 os.path.join(self.data_utils.path_manager.data_set_train_path, os.path.basename(_file_name)) 
)
 _count += 1
if _count >= top_k:
break
image_file_list.append(_image_file_list)
 image_utils.show_images_file(image_list=image_file_list, image_save_file=None, image_size=(200, 200), dpi=100)
 def model_fun(self,):

 raise NotImplementedError
def list_features(self, file_list: list = None, mode: ProcessMode = None)->(list, list):
if file_list is None and mode is not None:
if mode == ProcessMode.train:
input_list = self.data_utils.get_file_list(is_training=True)
 else:
input_list = self.data_utils.get_file_list(is_training=False)
 else:
input_list = file_list
input_fn = self.get_dataset_func(
 split_name=DatasetUtils.SPLIT_PREDICT, input_list=input_list,
num_epochs=1, shuffle=False, batch_size=self.batch_size, num_parallel_calls=2, prefetch_size=2, 
)
 self.logger.info("trying to list feature for {} file...".format(len(input_list)))
 classifier = self.get_classifier()
result_list = classifier.predict(input_fn=input_fn)
 feature_list = []
feature_file_list = []
count = 0
_steps_ = 100 if len(input_list)<= 1000 else 1000
for score in result_list:
feature_file_list.append(byte_to_string(score[self._features_filename_key]))
feature_list.append(score[self._features_embedding_key].reshape(1, self.params.dimension))
count += 1
if count % _steps_ == 0:
self.logger.info("calculated {} image...".format(count))
if count % _steps_ != 0:
self.logger.info("calculated {} image...".format(count))
self.logger.info("success to list feature for {} file!".format(len(input_list)))
 return feature_list, feature_file_list
def get_dataset_func(self, split_name, num_epochs=1, shuffle=True, batch_size=64, num_parallel_calls=2,
prefetch_size=2, shuffle_size=4, input_list=None):
if self.train_loss_hook is None:
raise ValueError("self.train_loss_hook cannot be null in this class!")
 def tf_decode_with_crop(file_name, label, offset_height, offset_width, target_height, target_width):
image_str = tf.read_file(file_name)
 image = tf.image.decode_jpeg(image_str, channels=3)
 image = tf_image_crop(image, offset_height, offset_width, target_height, target_width)
 image = tf.image.resize_images(tf.image.rgb_to_grayscale(image),
size=(self.image_tmp_size, self.image_tmp_size))
processed_images = whale_gray_preprocess_image(
 image, self.params.image_size, self.params.image_size,
is_training=bool(split_name == DatasetUtils.SPLIT_TRAIN),
mean_tf_func=whale_siamese_image_mean_tf)
 return {self._features_images_key: processed_images, self._features_filename_key: file_name}, label
def input_fn() :
if input_list:
file_list = input_list
else:
file_list = self.data_utils.get_file_list(
 is_training=bool(split_name == DatasetUtils.SPLIT_TRAIN),
shuffle=shuffle, num_epochs=num_epochs, batch_size=batch_size,
online_batch_count=self.params.online_batch_count)
 self.logger.info("info of file_list: len is {}.".format(len(file_list)))
 if split_name == DatasetUtils.SPLIT_TRAIN:
labels = [self.data_utils.get_label(image_file)for image_file in file_list]
else:
labels = [0] * len(file_list)
 bounding_boxes = [expend_bounding_box(self.data_utils.get_boxes(image_file)) for
image_file in file_list]
offset_height_list, offset_width_list, target_height_list, target_width_list = \
parse_bounding_boxes_list(bounding_boxes)
 dataset = tf.data.Dataset.from_tensor_slices(
(file_list, labels, offset_height_list, offset_width_list, target_height_list, target_width_list))
dataset = dataset.map(tf_decode_with_crop, num_parallel_calls=num_parallel_calls)
 dataset = dataset.prefetch(buffer_size=prefetch_size * batch_size)
 dataset = dataset.batch(batch_size)
 iterator = dataset.make_one_shot_iterator()
features, labels = iterator.get_next()
return features, labels
return input_fn
def map_search(self, only_validate: bool = False, clear_cache_after_exists: bool = True,
norm_type: NormType = NormType.all):
with TripletSearch(instance=self, data_utils=self.data_utils, norm_type=norm_type,
clear_cache_after_exists=clear_cache_after_exists)as search:
self.logger.info("trying to search with {}".format(self.__class__.__name__))
raw_submission_csv = self.data_utils.path_manager.submission_csv
try:
_time_start = time.time()
search.train()
base_name = "whale.sub_.{}".format(create_fake_random_string(length=8))
self.data_utils.path_manager.submission_csv = os.path.join(self.data_utils.path_manager.output_path,
base_name)
 map_5, map_1 = search.validate(data_percent=1.0)
 if only_validate:
self.logger.info("success to validate with {}, time cost {} seconds!".format(
 self.__class__.__name__, round(time.time() - _time_start, 2)))
 return
self.data_utils.path_manager.submission_csv = \
self.data_utils.path_manager.submission_csv + \
".{}.submission.csv".format(str(( map_5 + map_1)/ 2)[2:])
 search.test()
self.logger.info("success to search with {}, time cost {} seconds, submission save in {}".format(
 self.__class__.__name__, round(time.time() - _time_start, 2),
self.data_utils.path_manager.submission_csv))
finally:
self.data_utils.path_manager.submission_csv = raw_submission_csv
def combine_submission_csv(self):
tmp_submission_csv_list = glob.glob(
 os.path.join(self.data_utils.path_manager.output_path, "whale.sub_.*.submission.csv"))
if not tmp_submission_csv_list:
self.logger.warning("no submission file found!")
 return
self.logger.info("found tmp submission files: {}".format(tmp_submission_csv_list))
result_weight = {}
for csv_file in tmp_submission_csv_list:
try:
result_weight[csv_file] = float("0.{}".format(csv_file.split(".")[-3]))
except Exception as e:
self.logger.error(e)
 combine_csv(result_weight, out_file=os.path.join(self.data_utils.path_manager.output_path, "sub_ens.csv"))
self.logger.info("success to merge all submission file into sub_ens.csv")
 if self.data_utils.path_manager.is_kaggle:
for csv_file in tmp_submission_csv_list:
os.remove(csv_file)
 def train_with_predict(self, safe_max_batch_size=32, calc_every_epoch=1, shuffle_data_every_epoch=1,
predict_every_epoch=100, max_epoch=100, ignore_error_in_train: bool = True):
assert max_epoch >= predict_every_epoch >= shuffle_data_every_epoch
assert max_epoch >= calc_every_epoch >= shuffle_data_every_epoch
self.batch_size = safe_max_batch_size
def end_process_func() :
self.map_search(only_validate=False, clear_cache_after_exists=True)
 def loop_process(total_epoch, num_epoch):
if total_epoch % calc_every_epoch == 0:
if total_epoch == 0:
self.train(batch_size=safe_max_batch_size, num_epochs=1, shuffle=True, steps=1)
 self.data_utils.calc_feature(self.list_features, epoch_num=total_epoch)
 if total_epoch > 0 and not self.data_utils.path_manager.is_kaggle and \
os.path.abspath(self.TRAIN_CKPT_DIR ).find("/content/")> -1:
colab_save_file_func(train_dir=self.TRAIN_CKPT_DIR, logger=self.logger, daemon=False,
only_save_latest_checkpoint=True)
 self.train(batch_size=safe_max_batch_size, num_epochs=num_epoch, shuffle=True, steps=None)
 self.set_epoch_num(count=total_epoch + num_epoch)
 if total_epoch > 0 and total_epoch % predict_every_epoch == 0:
end_process_func()
estimator_iter_process(loop_process, iter_stop_time=self.timeout,
loop_process_min_epoch=shuffle_data_every_epoch,
loop_process_start_epoch=self.get_epoch_num() ,
end_process_func=end_process_func, loop_process_max_epoch=max_epoch,
ignore_error_in_loop_process=ignore_error_in_train, logger=self.logger)
 self.combine_submission_csv()
class TripletLossModelSiamese(TripletLossModelCNN):
def __init__(self, train_ckpt_dir, data_utils: WhaleDataUtils, timeout: int = int(3600 * 5),
pretrained_ckpt_file: str = None, keras_model: str = None):
super(TripletLossModelSiamese, self ).__init__(
 train_ckpt_dir=train_ckpt_dir,
data_utils=data_utils,
timeout=timeout,
pretrained_ckpt_file=None if keras_model else pretrained_ckpt_file, 
)
 self.keras_model_file = keras_model
if not self._checkpoint_exist and keras_model:
ckpt_file = keras_convert_model_to_estimator_ckpt(
 keras_model_path=keras_model, log_dir=self.TRAIN_CKPT_DIR, logger=self.logger)
 if not ckpt_file:
self.logger.warning("fail to convert keras model to estimator!")
 else:
self.train_restore_hook = InitFromPretrainedCheckpointHook(
 ckpt_file, exclusion_list=["global_step"], index_add=1, root_name=self.model_name)
 def get_dataset_func(self, split_name, num_epochs=1, shuffle=True, batch_size=64, num_parallel_calls=2,
prefetch_size=2, shuffle_size=4, input_list=None):
if self.train_loss_hook is None:
raise ValueError("self.train_loss_hook cannot be null in this class!")
 def tf_decode_with_crop(file_name, label, offset_height, offset_width, target_height, target_width):
image_str = tf.read_file(file_name)
 image = tf.image.decode_jpeg(image_str, channels=3)
 image = tf_image_crop(image, offset_height, offset_width, target_height, target_width)
 image = tf.image.resize_images(tf.image.rgb_to_grayscale(image),
size=(self.image_tmp_size, self.image_tmp_size))
processed_images = whale_gray_preprocess_image(
 image, self.params.image_size, self.params.image_size,
is_training=bool(split_name == DatasetUtils.SPLIT_TRAIN),
mean_tf_func=whale_siamese_image_mean_tf)
 return {self._features_images_key: processed_images, self._features_filename_key: file_name}, label
def input_fn() :
if input_list:
file_list = input_list
else:
file_list = self.data_utils.get_file_list(
 is_training=bool(split_name == DatasetUtils.SPLIT_TRAIN),
shuffle=shuffle, num_epochs=num_epochs, batch_size=batch_size,
online_batch_count=self.params.online_batch_count)
 self.logger.info("info of file_list: len is {}.".format(len(file_list)))
 if split_name == DatasetUtils.SPLIT_TRAIN:
labels = [self.data_utils.get_label(image_file)for image_file in file_list]
else:
labels = [0] * len(file_list)
 bounding_boxes = [expend_bounding_box(self.data_utils.get_boxes(image_file)) for
image_file in file_list]
offset_height_list, offset_width_list, target_height_list, target_width_list = \
parse_bounding_boxes_list(bounding_boxes)
 dataset = tf.data.Dataset.from_tensor_slices(
(file_list, labels, offset_height_list, offset_width_list, target_height_list, target_width_list))
dataset = dataset.map(tf_decode_with_crop, num_parallel_calls=num_parallel_calls)
 dataset = dataset.prefetch(buffer_size=prefetch_size * batch_size)
 dataset = dataset.batch(batch_size)
 iterator = dataset.make_one_shot_iterator()
features, labels = iterator.get_next()
return features, labels
return input_fn
def get_bone_net(self,):

 def sub_block(x, n_filter, **kwargs):
x = BatchNormalization()(x)
 y = x
y = Conv2D(n_filter,(1, 1), activation='relu', **kwargs )(y)
 y = BatchNormalization()(y)
 y = Conv2D(n_filter,(3, 3), activation='relu', **kwargs )(y)
 y = BatchNormalization()(y)
 y = Conv2D(K.int_shape(x)[-1],(1, 1), **kwargs )(
 y)
 y = Add()([x, y])
 y = Activation('relu' )(y)
 return y
def get_model_net(is_training: bool, img_shape: tuple,):
if is_training:
regul = regularizers.l2(0.0002)
 else:
regul = regularizers.l2(0)
 kwargs = {'padding': 'same', 'kernel_regularizer': regul}
inp = Input(shape=img_shape)
 x = Conv2D(64,(9, 9), strides=2, activation='relu', **kwargs )(inp)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 for _ in range(2):
x = BatchNormalization()(x)
 x = Conv2D(64,(3, 3), activation='relu', **kwargs )(x)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(128,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = sub_block(x, 64, **kwargs)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(256,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = sub_block(x, 64, **kwargs)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(384,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = sub_block(x, 96, **kwargs)
 x = MaxPooling2D(( 2, 2), strides=(2, 2))(x)
 x = BatchNormalization()(x)
 x = Conv2D(512,(1, 1), activation='relu', **kwargs )(x)
 for _ in range(4):
x = sub_block(x, 128, **kwargs)
 x = GlobalMaxPooling2D()(x)
 return Model(inputs=inp, outputs=x)
 return get_model_net
def model_fun(self,):

 def network(scope_name: str, features: dict, params: Params, is_training: bool = True, labels=None):
assert params.num_channels == 1
img_shape = [params.image_size, params.image_size, params.num_channels]
input_shape = [-1] + img_shape
inputs = tf.reshape(features[self._features_images_key], input_shape)
 with tf.variable_scope(scope_name, 'triplet_loss', [inputs]):
branch_model = self.get_bone_net()(is_training=is_training, img_shape=img_shape)
 embeddings = branch_model(inputs)
 return embeddings, {}
return tf_model_fn.tf_triplet_loss_model_fn(
 network=network,
scope_name=self.model_name,
features_embedding_key=self._features_embedding_key,
features_filename_key=self._features_filename_key,
get_learning_rate_func=self.get_learning_rate,
optimizer_type=self.optimizer_type,
logger=self.logger,
use_l2_normalize=False )<load_pretrained>
train_vectors = tfidf_vectorizer.fit_transform(train_df["text"])
 test_vectors = tfidf_vectorizer.transform(test_df["text"] )
!mkdir -p./keras
!cp.. /input/piotte/mpiotte-standard.model./keras/
!cp.. /input/whale-triplet-pretrained-model/tripletk/tripletK triplet -R
init_logger()
path_manager = PathManager("kaggle")
 data_utils = WhaleDataUtils(path_manager=path_manager, gen_data_setting={
"x_train_num": 4,
"ignore_blank_prob": 0.9,
"ignore_single_prob": 0.5,
"stop_calc_feature": False,
"gen_data_by_random_prob": 0,
"use_norm_when_calc_apn": True,
})
 estimator = TripletLossModelSiamese(
 train_ckpt_dir="./triplet",
data_utils=data_utils,
timeout=int(5 * 3600),
pretrained_ckpt_file=None,
keras_model=None, 
)
 estimator.map_search(only_validate=False, clear_cache_after_exists=True)
 estimator.combine_submission_csv()
estimator.show_embedding(
 mode=ProcessMode.train,
count=1000,
remove_log_dir_if_exists=True,
norm_type=NormType.all, 
)
<predict_on_test>
feature_cols = ['keyword', 'location']
X = train_df[feature_cols]
y = train_df.target
one_hot_encoded_training_predictors = pd.get_dummies(X)
 clf = RandomForestClassifier(n_estimators = 100)
 scores = model_selection.cross_val_score(clf, one_hot_encoded_training_predictors, y, cv=5, scoring="f1")
 scores
estimator.show_predict_result(count=10, top_k=5)
 estimator.show_predict_result(count=20, top_k=3)
<install_modules>
clf.fit(one_hot_encoded_training_predictors, y )
!rm *.pkl
!rm./keras/ -R
!pip uninstall aiohttp faiss-prebuilt pyxtools pymltools -y
!apt remove -y libopenblas-base libomp-dev 
<import_modules>
clf = linear_model.RidgeClassifier()
from keras.layers import Dense, Flatten, Dropout, Lambda, Input, Concatenate, concatenate
from keras.models import Model
from keras.applications import *
from keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from keras.callbacks import ModelCheckpoint
from keras import regularizers<define_variables>
scores = model_selection.cross_val_score(clf, train_vectors, train_df["target"], cv=10, scoring="f1")
 scores
filenames = os.listdir(".. /input/train/train")
 labels = []
for file in filenames:
category = file.split('.')[0]
if category == 'cat':
labels.append('cat')
 else:
labels.append('dog' )<split>
clf.fit(train_vectors, train_df["target"] )
df = pd.DataFrame({
'filename': filenames,
'label': labels
})
 train_df, validation_df = train_test_split(df, test_size=0.1, random_state = 42)
 train_df = train_df.reset_index(drop=True)
 validation_df = validation_df.reset_index(drop=True)
<define_variables>
parameters = {
'gamma': [0.7, 1, 'auto', 'scale']
}
clf = GridSearchCV(SVC(kernel='rbf'), parameters, cv=5, n_jobs=-1, scoring="f1" ).fit(train_vectors, train_df["target"] )
batch_size = 64
train_num = len(train_df)
 validation_num = len(validation_df )<create_dataframe>
clf.best_estimator_
def two_image_generator(generator, df, directory, batch_size,
x_col = 'filename', y_col = None, model = None, shuffle = False,
img_size1 =(224, 224), img_size2 =(299,299)) :
gen1 = generator.flow_from_dataframe(
 df,
directory,
x_col = x_col,
y_col = y_col,
target_size = img_size1,
class_mode = model,
batch_size = batch_size,
shuffle = shuffle,
seed = 1)
 gen2 = generator.flow_from_dataframe(
 df,
directory,
x_col = x_col,
y_col = y_col,
target_size = img_size2,
class_mode = model,
batch_size = batch_size,
shuffle = shuffle,
seed = 1)
 while True:
X1i = gen1.next()
X2i = gen2.next()
if y_col:
yield [X1i[0], X2i[0]], X1i[1]
else:
yield [X1i, X2i] 
<prepare_x_and_y>
clf.best_score_
<load_pretrained>
sample_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 sample_submission["target"] = clf.predict(test_vectors)
 df = pd.DataFrame({'text' : test_df['text'], 'prediction' : sample_submission["target"]} )
train_aug_datagen = ImageDataGenerator(
 rotation_range = 20,
shear_range = 0.1,
zoom_range = 0.2,
width_shift_range = 0.1,
height_shift_range = 0.1,
horizontal_flip = True 
)
 train_generator = two_image_generator(train_aug_datagen, train_df, '.. /input/train/train/',
batch_size = batch_size, y_col = 'label',
model = 'binary', shuffle = True )<define_search_model>
sample_submission.to_csv("submission1.csv", index=False )
validation_datagen = ImageDataGenerator()
validation_generator = two_image_generator(validation_datagen, validation_df,
'.. /input/train/train/', batch_size = batch_size,
y_col = 'label',model = 'binary', shuffle = True )<choose_model_class>
print(tf.__version__)
def create_base_model(MODEL, img_size, lambda_fun = None):
inp = Input(shape =(img_size[0], img_size[1], 3))
x = inp
if lambda_fun:
x = Lambda(lambda_fun )(x)
 base_model = MODEL(input_tensor = x, weights = 'imagenet', include_top = False, pooling = 'avg')
 model = Model(inp, base_model.output)
 return model<choose_model_class>
X = train_df["text"]
y = train_df["target"]
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.3, random_state=42 )
model1 = create_base_model(vgg16.VGG16,(224, 224), vgg16.preprocess_input)
 model2 = create_base_model(resnet50.ResNet50,(224, 224), resnet50.preprocess_input)
 model3 = create_base_model(inception_v3.InceptionV3,(299, 299), inception_v3.preprocess_input)
 model1.trainable = False
model2.trainable = False
model3.trainable = False
inpA = Input(shape =(224, 224, 3))
inpB = Input(shape =(299, 299, 3))
out1 = model1(inpA)
 out2 = model2(inpA)
 out3 = model3(inpB)
 x = Concatenate()([out1, out2, out3])
 x = Dropout(0.6 )(x)
 x = Dense(1, activation='sigmoid' )(x)
 multiple_pretained_model = Model([inpA, inpB], x)
 multiple_pretained_model.compile(loss = 'binary_crossentropy',
optimizer = 'rmsprop',
metrics = ['accuracy'])
 multiple_pretained_model.summary()<choose_model_class>
vocab_size = 10000
embedding_dim = 16
max_length = 30
trunc_type='post'
padding_type='post'
oov_tok = "<OOV>"
checkpointer = ModelCheckpoint(filepath='dogcat.weights.best.hdf5', verbose=1,
save_best_only=True, save_weights_only=True )<train_model>
tokenizer = Tokenizer(num_words = vocab_size, oov_token=oov_tok)
 tokenizer.fit_on_texts(X_train)
 word_index = tokenizer.word_index
sequences = tokenizer.texts_to_sequences(X_train)
 padded = pad_sequences(sequences,maxlen=max_length, padding=padding_type, truncating=trunc_type)
 testing_sequences = tokenizer.texts_to_sequences(X_val)
 testing_padded = pad_sequences(testing_sequences,maxlen=max_length, padding=padding_type, truncating=trunc_type )
multiple_pretained_model.fit_generator(
 train_generator,
epochs = 5,
steps_per_epoch = train_num // batch_size,
validation_data = validation_generator,
validation_steps = validation_num // batch_size,
verbose = 1,
callbacks = [checkpointer]
)<load_pretrained>
reverse_word_index = dict([(value, key)for(key, value)in word_index.items() ])
 def decode_sentence(text):
return ' '.join([reverse_word_index.get(i, '?')for i in text])
 print(decode_sentence(padded[0]))
multiple_pretained_model.load_weights('dogcat.weights.best.hdf5' )<define_variables>
!wget --no-check-certificate \
https://storage.googleapis.com/laurencemoroney-blog.appspot.com/glove.6B.100d.txt \
-O /tmp/glove.6B.100d.txt
embeddings_index = {};
vocab_size=len(word_index)
 embedding_dim = 100
with open('/tmp/glove.6B.100d.txt')as f:
for line in f:
values = line.split() ;
word = values[0];
coefs = np.asarray(values[1:], dtype='float32');
embeddings_index[word] = coefs;
embeddings_matrix = np.zeros(( vocab_size+1, embedding_dim)) ;
for word, i in word_index.items() :
embedding_vector = embeddings_index.get(word);
if embedding_vector is not None:
embeddings_matrix[i] = embedding_vector;
test_filenames = os.listdir(".. /input/test/test")
 test_df = pd.DataFrame({
'filename': test_filenames
})
 num_test = len(test_df)
 test_datagen = ImageDataGenerator()
test_generator = two_image_generator(test_datagen, test_df, '.. /input/test/test/', batch_size = batch_size )<predict_on_test>
model = tf.keras.Sequential([
tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),
tf.keras.layers.GlobalAveragePooling1D() ,
tf.keras.layers.Dense(24, activation='relu'),
tf.keras.layers.Dense(1, activation='sigmoid')
 ])
 model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy',tf.keras.metrics.AUC() ])
 model.summary()
prediction = multiple_pretained_model.predict_generator(test_generator,
steps=np.ceil(num_test/batch_size))
prediction = prediction.clip(min = 0.005, max = 0.995 )<save_to_csv>
num_epochs = 3
history = model.fit(padded, y_train, epochs=num_epochs, validation_data=(testing_padded, y_val))
submission_df = pd.read_csv('.. /input/sample_submission.csv')
 for i, fname in enumerate(test_filenames):
index = int(fname[fname.rfind('/')+1:fname.rfind('.')])
 submission_df.at[index-1, 'label'] = prediction[i]
submission_df.to_csv('submission.csv', index=False )<import_modules>
model_loss = pd.DataFrame(model.history.history)
 model_loss.head()
import numpy as np
import pandas as pd
import os
from fastai.vision import *<define_variables>
testing_sequences2 = tokenizer.texts_to_sequences(test_df.text)
 testing_padded2 = pad_sequences(testing_sequences2, maxlen=max_length, padding=padding_type, truncating=trunc_type )
path = Path('.. /input' )<set_options>
probabilities = model.predict(testing_padded2 )
path.ls()<define_variables>
predictions =(probabilities > 0.5 ).astype(int)
 predictions = np.ndarray.flatten(predictions)
 pd.value_counts(predictions )
path_img = path/'train'<features_selection>
original_test_df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 df = pd.DataFrame({'text' : original_test_df['text'],'cleaned_text' : test_df['text'], 'prediction' : predictions,'probabilities' : np.ndarray.flatten(probabilities)})
 df.to_csv("test_df.csv", index=False )
get_image_files(path_img)[:5]<define_variables>
sample_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 sample_submission["target"] = predictions
sample_submission.to_csv("submission.csv", index=False )
np.random.seed(42)
 size = 224
bs = 64
num_workers = 0
pat = r'/([^/.]+ ).\d+.jpg$'<categorify>
import tensorflow_hub as hub
import lightgbm as lgb
from lightgbm import LGBMClassifier
tfms = get_transforms()
data =(ImageItemList.from_folder(path_img)
.random_split_by_pct() 
.label_from_re(pat)
.add_test_folder('.. /test')
.transform(tfms, size=size)
.databunch(bs=bs, num_workers=num_workers)
.normalize(imagenet_stats))<define_variables>
module_url = "https://tfhub.dev/google/nnlm-en-dim128/2"
embed = hub.KerasLayer(module_url)
 embeddings = embed(["A long sentence.", "single-word",
"http://example.com"])
 print(embeddings.shape )
data.show_batch(rows=3, figsize=(7,6))<choose_model_class>
embed = hub.load("https://tfhub.dev/google/universal-sentence-encoder/3" )
learn = create_cnn(data, models.resnet50, metrics=accuracy, model_dir='/tmp/models' )<train_model>
X_train_embeddings = embed(train_df.text.values)
 X_test_embeddings = embed(test_df.text.values )
learn.fit_one_cycle(4 )<save_model>
params = {
'learning_rate': 0.04,
'n_estimators': 1500,
'colsample_bytree': 0.4,
'metric':'auc'
}
learn.save('stage-1' )<train_model>
text_clf = LGBMClassifier(**params )
learn.fit_one_cycle(2, max_lr=slice(1e-6,1e-4))<save_model>
text_clf.fit(X_train_embeddings['outputs'][:5000,:], train_df.target.values[:5000],
eval_set=[(X_train_embeddings['outputs'][:5000,:], train_df.target.values[:5000]), 
(X_train_embeddings['outputs'][5000:,:], train_df.target.values[5000:])],
verbose=200, early_stopping_rounds=20, 
)
learn.save('stage-2' )<find_best_params>
text_clf.fit(X_train_embeddings['outputs'][:5000,:], train_df.target.values[:5000])
 Y_pred = text_clf.predict(X_train_embeddings['outputs'][5000:] )
interp = ClassificationInterpretation.from_learner(learn)
 losses,idxs = interp.top_losses()
len(data.valid_ds)==len(losses)==len(idxs )<predict_on_test>
print(metrics.classification_report(train_df.target[5000:], Y_pred, digits=3),)
 print(metrics.confusion_matrix(train_df.target[5000:], Y_pred))
preds, y = learn.get_preds(ds_type=DatasetType.Test )<prepare_output>
text_clf.fit(X_train_embeddings['outputs'], train_df.target.values)
 pred_test = text_clf.predict(X_test_embeddings['outputs'] )
dog_preds = preds[:,1]<create_dataframe>
df = pd.DataFrame({'cleaned_text' : test_df['text'], 'prediction' : pred_test})
 df.head(20 )
<feature_engineering><EOS> 
sample_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 sample_submission["target"] = pred_test
sample_submission.to_csv("submission.csv", index=False )