Kaggle competition: Humpback Whale Identification
<SOS> metric: MAP@{K} Kaggle data source: humpback-whale-identification<load_pretrained>
!pip install fastai==0.7.0 --no-deps
!pip install torch==0.4.1 torchvision==0.2.1
histories = []
steps = 0
if isfile('.. /input/humpback-whale-identification-model-files/mpiotte-standard.model'):
print('pretrained exist.')
 tmp = keras.models.load_model('.. /input/humpback-whale-identification-model-files/mpiotte-standard.model')
 model.set_weights(tmp.get_weights())
 else:
make_steps(10, 1000)
 ampl = 100.0
for _ in range(2):
print('noise ampl.= ', ampl)
 make_steps(5, ampl)
 ampl = max(1.0, 100 ** -0.1 * ampl)
<load_pretrained>
MODEL_NAME = 'Resnet101'
TRAIN = '.. /input/train/'
TEST = '.. /input/test/'
LABELS = '.. /input/train.csv'
SAMPLE_SUB = '.. /input/sample_submission.csv'
arch = resnet101
num_workers = 4
def prepare_submission(threshold, filename):

 vtop = 0
vhigh = 0
pos = [0, 0, 0, 0, 0, 0]
with open(filename, 'wt', newline='
')as f:
f.write('Image,Id
')
 for i, p in enumerate(tqdm(submit)) :
t = []
s = set()
a = score[i, :]
for j in list(reversed(np.argsort(a))):
h = known[j]
if a[j] < threshold and new_whale not in s:
pos[len(t)] += 1
s.add(new_whale)
 t.append(new_whale)
 if len(t)== 5: break;
for w in h2ws[h]:
assert w != new_whale
if w not in s:
if a[j] > 1.0:
vtop += 1
elif a[j] >= threshold:
vhigh += 1
s.add(w)
 t.append(w)
 if len(t)== 5: break;
if len(t)== 5: break;
if new_whale not in s: pos[5] += 1
assert len(t)== 5 and len(s)== 5
f.write(p + ',' + ' '.join(t[:5])+ '
')
 return vtop, vhigh, pos<feature_engineering>
df = pd.read_csv(LABELS ).set_index('Image')
 new_whale_df = df[df.Id == "new_whale"]
train_df = df[~(df.Id == "new_whale")]
unique_labels = np.unique(train_df.Id.values)
 labels_dict = dict()
labels_list = []
for i in range(len(unique_labels)) :
labels_dict[unique_labels[i]] = i
labels_list.append(unique_labels[i])
 print("Number of classes: {}".format(len(unique_labels)))
 train_df.Id = train_df.Id.apply(lambda x: labels_dict[x])
 train_labels = np.asarray(train_df.Id.values)
 test_names = [f for f in os.listdir(TEST)]
tic = time.time()
h2ws = {}
for p, w in tagged.items() :
if w != new_whale:
h = p2h[p]
if h not in h2ws: h2ws[h] = []
if w not in h2ws[h]: h2ws[h].append(w)
 known = sorted(list(h2ws.keys()))
h2i = {}
for i, h in enumerate(known): h2i[h] = i
fknown = branch_model.predict_generator(FeatureGen(known), max_queue_size=20, workers=10, verbose=0)
 fsubmit = branch_model.predict_generator(FeatureGen(submit), max_queue_size=20, workers=10, verbose=0)
 score = head_model.predict_generator(ScoreGen(fknown, fsubmit), max_queue_size=20, workers=10, verbose=0)
 score = score_reshape(score, fknown, fsubmit)
 prepare_submission(0.99, 'submission_for_em.csv')
 toc = time.time()
print("Submission time: ",(toc - tic)/ 60.) 
<define_variables>
train_df['image_name'] = train_df.index
rs = np.random.RandomState(42)
 perm = rs.permutation(len(train_df))
tr_n = train_df['image_name'].values
val_n = train_df['image_name'].values[perm][:1000]
print('Train/val:', len(tr_n), len(val_n))
print('Train classes', len(train_df.loc[tr_n].Id.unique()))
print('Val classes', len(train_df.loc[val_n].Id.unique()))
sub_files = ['.. /input/em-whale/submission_0.csv',
'.. /input/em-whale/submission_7.csv',
'.. /input/em-whale/submission_8.csv',
'.. /input/em-whale/submission_3.csv']
sub_weight = [0.9001**2,
0.7719**2,
0.8229**2,
0.8903**2]
Hlabel = 'Image'
Htarget = 'Id'
npt = 6
place_weights = {}
for i in range(npt):
place_weights[i] =(1 /(i + 1))
print(place_weights)
 lg = len(sub_files)
 sub = [None]*lg
for i, file in enumerate(sub_files):
print("Reading {}: w={} - {}".format(i, sub_weight[i], file))
reader = csv.DictReader(open(file,"r"))
sub[i] = sorted(reader, key=lambda d: str(d[Hlabel]))
out = open("submission.csv", "w", newline='')
 writer = csv.writer(out)
 writer.writerow([Hlabel,Htarget])
 for p, row in enumerate(sub[0]):
target_weight = {}
for s in range(lg):
row1 = sub[s][p]
for ind, trgt in enumerate(row1[Htarget].split(' ')) :
target_weight[trgt] = target_weight.get(trgt,0)+(place_weights[ind]*sub_weight[s])
 tops_trgt = sorted(target_weight, key=target_weight.get, reverse=True)[:npt]
writer.writerow([row1[Hlabel], " ".join(tops_trgt)])
 out.close()<define_variables>
class HWIDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.train_df = train_df
super().__init__(fnames, transform, path)
 def get_x(self, i):
img = open_image(os.path.join(self.path, self.fnames[i]))
img = cv2.resize(img,(self.sz, self.sz))
return img
def get_y(self, i):
if(self.path == TEST): return 0
return self.train_df.loc[self.fnames[i]]['Id']
def get_c(self):
return len(unique_labels )
TRAIN_DF = '.. /input/humpback-whale-identification/train.csv'
SUB_Df = '.. /input/humpback-whale-identification/sample_submission.csv'
TRAIN = '.. /input/humpback-whale-identification/train/'
TEST = '.. /input/humpback-whale-identification/test/'
P2H = '.. /input/metadata/p2h.pickle'
P2SIZE = '.. /input/metadata/p2size.pickle'
BB_DF = ".. /input/metadata/bounding_boxes.csv" 
<load_from_csv>
class RandomLighting(Transform):
def __init__(self, b, c, tfm_y=TfmType.NO):
super().__init__(tfm_y)
 self.b, self.c = b, c
def set_state(self):
self.store.b_rand = rand0(self.b)
 self.store.c_rand = rand0(self.c)
 def do_transform(self, x, is_y):
if is_y and self.tfm_y != TfmType.PIXEL: return x
b = self.store.b_rand
c = self.store.c_rand
c = -1 /(c - 1)if c < 0 else c + 1
x = lighting(x, b, c)
 return x
def get_data(sz, batch_size):

 aug_tfms = [RandomRotateZoom(deg=20, zoom=2, stretch=1),
RandomLighting(0.2, 0.2, tfm_y=TfmType.NO),
RandomBlur(blur_strengths=3,tfm_y=TfmType.NO),
RandomFlip(tfm_y=TfmType.NO)]
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.NO,
aug_tfms=aug_tfms)
 ds = ImageData.get_ds(HWIDataset,(tr_n[:-(len(tr_n)% batch_size)], TRAIN), 
(val_n, TRAIN), tfms, test=(test_names, TEST))
md = ImageData("./", ds, batch_size, num_workers=num_workers, classes=None)
 return md
!pip install lap
tagged = dict([(p, w)for _, p, w in read_csv(TRAIN_DF ).to_records() ])
 submit = [p for _, p, _ in read_csv(SUB_Df ).to_records() ]
join = list(tagged.keys())+ submit
len(tagged),len(submit),len(join),list(tagged.items())[:5],submit[:5]<import_modules>
image_size = 299
batch_size = 32
md = get_data(image_size, batch_size)
 extra_fc_layers_size = []
learn = ConvLearner.pretrained(arch, md, xtra_fc=extra_fc_layers_size)
 learn.opt_fn = optim.Adam
import gzip
import pickle
from lap import lapjv
from math import sqrt
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from imagehash import phash
from keras.utils import Sequence
import time<import_modules>
print('Number of layer groups:', len(learn.get_layer_groups()), '\t(first 2 groups is pretrained backbone)')
 print('This is our extra thin on top of the backbone Resnet50 architecture:')
 learn.get_layer_groups() [2]
import platform
import random
import keras
import sys
from scipy.ndimage import affine_transform
from keras import backend as K
from keras.preprocessing.image import img_to_array,array_to_img<set_options>
base_lr = 5e-4
fc_lr = 1e-3
lrs = [base_lr, base_lr, fc_lr]
learn.fit(lrs=lrs, n_cycle=2, cycle_len=None)
 learn.unfreeze()
learn_bn_unfreeze()
learn.fit(lrs, n_cycle=4, cycle_len=1, cycle_mult=2 )
with open(P2H, 'rb')as f:
p2h = pickle.load(f)
 h2ps = {}
for p, h in p2h.items() :
if h not in h2ps: h2ps[h] = []
if p not in h2ps[h]: h2ps[h].append(p)
 with open(P2SIZE, 'rb')as f:
p2size = pickle.load(f)
 p2bb = pd.read_csv(BB_DF ).set_index("Image")
 old_stderr = sys.stderr
sys.stderr = open('/dev/null' if platform.system() != 'Windows' else 'nul', 'w')
 sys.stderr = old_stderr
img_shape =(384, 384, 1)
 anisotropy = 2.15
crop_margin = 0.1<categorify>
base_lr = 1e-4
fc_lr = 1e-3
lrs = [base_lr, base_lr, fc_lr]
learn.fit(lrs, n_cycle=6, cycle_len=1 )
def prefer(ps):
if len(ps)== 1: return ps[0]
best_p = ps[0]
best_s = p2size[best_p]
for i in range(1, len(ps)) :
p = ps[i]
s = p2size[p]
if s[0] * s[1] > best_s[0] * best_s[1]:
best_p = p
best_s = s
return best_p
h2p = {}
for h, ps in h2ps.items() :
h2p[h] = prefer(ps)
 len(h2p), list(h2p.items())[:5]<normalization>
best_th = 0.38
preds_t,y_t = learn.TTA(is_test=True,n_aug=8)
 preds_t = np.stack(preds_t, axis=-1)
 preds_t = np.exp(preds_t)
 preds_t = preds_t.mean(axis=-1)
 preds_t = np.concatenate([np.zeros(( preds_t.shape[0],1)) +best_th, preds_t],axis=1)
 np.save('preds.npy', preds_t )
<prepare_x_and_y><EOS> 
sample_df = pd.read_csv(SAMPLE_SUB)
 sample_list = list(sample_df.Image)
 labels_list = ["new_whale"]+labels_list
pred_list = [[labels_list[i] for i in p.argsort() [-5:][::-1]] for p in preds_t]
pred_dic = dict(( key, value)for(key, value)in zip(learn.data.test_ds.fnames,pred_list))
pred_list_cor = [' '.join(pred_dic[id])for id in sample_list]
df = pd.DataFrame({'Image':sample_list,'Id': pred_list_cor})
 df.to_csv('submission_{}.csv'.format(MODEL_NAME), header=True, index=False)
 df.head()