Kaggle competition: Humpback Whale Identification Challenge
<SOS> metric: MAP@{K} Kaggle data source: whale-categorization-playground<save_to_csv>
from collections import defaultdict
import pandas as pd
import numpy as np
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import normalize
from scipy.stats import logistic
from os.path import join
from tqdm import tqdm
from PIL import Image
from keras import backend as K
from keras.models import Model
from keras.optimizers import Adam
from keras.layers import Input, Dense, Dropout, Lambda, Convolution2D, MaxPooling2D, Flatten
from keras.losses import categorical_crossentropy
from keras.callbacks import ModelCheckpoint, EarlyStopping
from keras.applications.resnet50 import ResNet50, preprocess_input
import os
import matplotlib.pyplot as plt
submissions.to_csv('submission.csv', index=False)
 print('--Done--' )<load_from_csv>
batch_size = 24
embedding_dim = 50
image_size = 224
path_base = '.. /input/whale-categorization-playground/'
path_train = join(path_base,'train')
 path_test = join(path_base,'test')
 path_model = join(path_base,'MyModel.hdf5')
 path_csv = '.. /input/whale-categorization-playground/train.csv'
dirname = '/kaggle/input/covid19-global-forecasting-week-2/'
trainname = 'train.csv'
trainpath = dirname + trainname
traindata_raw = pd.read_csv(trainpath)
 traindata = collections.defaultdict(lambda :collections.defaultdict(list))
traindata_country = collections.defaultdict(lambda :collections.defaultdict(lambda :[0,0]))
traindata_state = collections.defaultdict(lambda :collections.defaultdict(lambda :[0,0]))
traindata_country_onlymainland = collections.defaultdict(lambda :collections.defaultdict(lambda :[0,0]))
for i in range(len(traindata_raw)) :
country = traindata_raw['Country_Region'][i]
state = traindata_raw['Province_State'][i]
date = traindata_raw['Date'][i]
cases = traindata_raw['ConfirmedCases'][i]
fata = traindata_raw['Fatalities'][i]
traindata[country][date].append(( state, cases, fata))
traindata_country[country][date][0] += cases
traindata_country[country][date][1] += fata
if state != state:
traindata_country_onlymainland[country][date][0] += cases
traindata_country_onlymainland[country][date][1] += fata
state_ = country + ':' + state if state == state else country + ':'
traindata_state[state_][date][0] += cases
traindata_state[state_][date][1] += fata 
<define_search_model>
class sample_gen(object):
def __init__(self, file_class_mapping, other_class = "new_whale"):
self.file_class_mapping= file_class_mapping
self.class_to_list_files = defaultdict(list)
 self.list_other_class = []
self.list_all_files = list(file_class_mapping.keys())
 self.range_all_files = list(range(len(self.list_all_files)))
 for file, class_ in file_class_mapping.items() :
if class_ == other_class:
self.list_other_class.append(file)
 else:
self.class_to_list_files[class_].append(file)
 self.list_classes = list(set(self.file_class_mapping.values()))
self.range_list_classes= range(len(self.list_classes))
self.class_weight = np.array([len(self.class_to_list_files[class_])for class_ in self.list_classes])
 self.class_weight = self.class_weight/np.sum(self.class_weight)
 def get_sample(self):
class_idx = np.random.choice(self.range_list_classes, 1, p=self.class_weight)[0]
examples_class_idx = np.random.choice(range(len(self.class_to_list_files[self.list_classes[class_idx]])) , 2)
 positive_example_1, positive_example_2 = \
self.class_to_list_files[self.list_classes[class_idx]][examples_class_idx[0]],\
self.class_to_list_files[self.list_classes[class_idx]][examples_class_idx[1]]
negative_example = None
while negative_example is None or self.file_class_mapping[negative_example] == \
self.file_class_mapping[positive_example_1]:
negative_example_idx = np.random.choice(self.range_all_files, 1)[0]
negative_example = self.list_all_files[negative_example_idx]
return positive_example_1, negative_example, positive_example_2
def read_and_resize(filepath):
im = Image.open(( filepath)).convert('RGB')
 im = im.resize(( image_size, image_size))
return np.array(im, dtype="float32")
 def augment(im_array):
if np.random.uniform(0, 1)> 0.9:
im_array = np.fliplr(im_array)
 return im_array
def gen(triplet_gen):
while True:
list_positive_examples_1 = []
list_negative_examples = []
list_positive_examples_2 = []
for i in range(batch_size):
positive_example_1, negative_example, positive_example_2 = triplet_gen.get_sample()
path_pos1 = join(path_train, positive_example_1)
 path_neg = join(path_train, negative_example)
 path_pos2 = join(path_train, positive_example_2)
 positive_example_1_img = read_and_resize(path_pos1)
 negative_example_img = read_and_resize(path_neg)
 positive_example_2_img = read_and_resize(path_pos2)
 positive_example_1_img = augment(positive_example_1_img)
 negative_example_img = augment(negative_example_img)
 positive_example_2_img = augment(positive_example_2_img)
 list_positive_examples_1.append(positive_example_1_img)
 list_negative_examples.append(negative_example_img)
 list_positive_examples_2.append(positive_example_2_img)
 A = preprocess_input(np.array(list_positive_examples_1))
B = preprocess_input(np.array(list_positive_examples_2))
C = preprocess_input(np.array(list_negative_examples))
label = None
yield({'anchor_input': A, 'positive_input': B, 'negative_input': C}, label )
polynomial = PolynomialFeatures(degree = 5)
 poly_model = LinearRegression()
predictdata_state = collections.defaultdict(lambda :[[],[]])
 for state in traindata_state:
state_history = [[],[]]
for date in traindata_state[state]:
state_history[0].append(traindata_state[state][date][0])
 state_history[1].append(traindata_state[state][date][1])
 i = 0
for perspective in state_history:
perspective_train = [i for i in perspective[:] if i!=0]
perspective = [i for i in perspective if i!=0]
x_all = np.array(range(1, len(perspective)+ 31))
x = x_all[:len(perspective_train)]
y = np.array(perspective_train)
 if len(perspective_train)< 3:
if not perspective_train:
predict_y = np.zeros(len(x_all[len(perspective_train):]))
else:
predict_y = np.array([perspective_train[-1]]*(len(x_all[len(perspective_train):])))
 predictdata_state[state][i] = predict_y
i =(i+1)% 2
continue
if True:
def funmi(x, a, b, c):
return(x/a)**b + c
def funexp(x,a,b,c):
return(a**(x + b)) * c
def funsigmoid(x, a, b, c):
s = a /(1 + np.exp(-(x + b)/ c))
return s
popt, pcov = curve_fit(funsigmoid, x, y, maxfev = 10000)
 predict_x = x_all[len(perspective_train):]
predict_y = [funsigmoid(i, popt[0], popt[1], popt[2])for i in predict_x]
if predict_y[0] < perspective_train[-1]:
predict_y = [i +(perspective_train[-1] - predict_y[0])for i in predict_y]
predict_y = [round(i)for i in predict_y]
predictdata_state[state][i] = predict_y
else:
x = x.reshape(-1,1)
 x_transformed = polynomial.fit_transform(x)
 poly_model.fit(x_transformed, y)
 predict_x = x_all[len(perspective_train):]
predict_y = poly_model.predict(polynomial.fit_transform(np.array(predict_x ).reshape(-1,1)))
 predict_y = [round(i)for i in predict_y]
predictdata_state[state][i] = predict_y
i =(i+1)% 2
print(predictdata_state)
<load_from_csv>
def triplet_loss(inputs, dist='sqeuclidean', margin='maxplus'):
anchor, positive, negative = inputs
positive_distance = K.square(anchor - positive)
 negative_distance = K.square(anchor - negative)
 if dist == 'euclidean':
positive_distance = K.sqrt(K.sum(positive_distance, axis=-1, keepdims=True))
negative_distance = K.sqrt(K.sum(negative_distance, axis=-1, keepdims=True))
elif dist == 'sqeuclidean':
positive_distance = K.sum(positive_distance, axis=-1, keepdims=True)
 negative_distance = K.sum(negative_distance, axis=-1, keepdims=True)
 loss = positive_distance - negative_distance
if margin == 'maxplus':
loss = K.maximum(0.0, 1 + loss)
 elif margin == 'softplus':
loss = K.log(1 + K.exp(loss))
return K.mean(loss)
 def triplet_loss_np(inputs, dist='sqeuclidean', margin='maxplus'):
anchor, positive, negative = inputs
positive_distance = np.square(anchor - positive)
 negative_distance = np.square(anchor - negative)
 if dist == 'euclidean':
positive_distance = np.sqrt(np.sum(positive_distance, axis=-1, keepdims=True))
negative_distance = np.sqrt(np.sum(negative_distance, axis=-1, keepdims=True))
elif dist == 'sqeuclidean':
positive_distance = np.sum(positive_distance, axis=-1, keepdims=True)
 negative_distance = np.sum(negative_distance, axis=-1, keepdims=True)
 loss = positive_distance - negative_distance
if margin == 'maxplus':
loss = np.maximum(0.0, 1 + loss)
 elif margin == 'softplus':
loss = np.log(1 + np.exp(loss))
return np.mean(loss)
 def check_loss() :
batch_size = 10
shape =(batch_size, 4096)
 p1 = normalize(np.random.random(shape))
n = normalize(np.random.random(shape))
p2 = normalize(np.random.random(shape))
input_tensor = [K.variable(p1), K.variable(n), K.variable(p2)]
out1 = K.eval(triplet_loss(input_tensor))
input_np = [p1, n, p2]
out2 = triplet_loss_np(input_np)
 assert out1.shape == out2.shape
print(np.linalg.norm(out1))
print(np.linalg.norm(out2))
print(np.linalg.norm(out1-out2))
testname = 'test.csv'
testpath = dirname + testname
testdata = pd.read_csv(testpath)
 submname = 'submission.csv'
submpath = dirname + submname
submdata = pd.read_csv(submpath)
 res = []
ind = 0
found = False
for i in range(len(testdata)) :
forecastid = testdata['ForecastId'][i]
state = testdata['Province_State'][i]
country = testdata['Country_Region'][i]
date = testdata['Date'][i]
state_ = country + ':' + state if state == state else country + ':'
if date == '2020-03-19':
found = True
if date == '2020-04-01':
found = False
ind = 0
if found:
res.append(traindata_state[state_][date])
 else:
res.append([int(predictdata_state[state_][0][ind]), int(predictdata_state[state_][1][ind])])
 ind += 1
subm_cases = [i[0] for i in res]
subm_fata = [i[1] for i in res]
submdata['ConfirmedCases'] = subm_cases
submdata['Fatalities'] = subm_fata
submdata.to_csv('submission.csv', index = False )<import_modules>
def GetModel() :
base_model = ResNet50(weights='imagenet', include_top=False, pooling='max')
 for layer in base_model.layers:
layer.trainable = False
x = base_model.output
x = Dropout(0.6 )(x)
 x = Dense(embedding_dim )(x)
 x = Lambda(lambda x: K.l2_normalize(x,axis=1))(x)
 embedding_model = Model(base_model.input, x, name="embedding")
 input_shape =(image_size, image_size, 3)
 anchor_input = Input(input_shape, name='anchor_input')
 positive_input = Input(input_shape, name='positive_input')
 negative_input = Input(input_shape, name='negative_input')
 anchor_embedding = embedding_model(anchor_input)
 positive_embedding = embedding_model(positive_input)
 negative_embedding = embedding_model(negative_input)
 inputs = [anchor_input, positive_input, negative_input]
outputs = [anchor_embedding, positive_embedding, negative_embedding]
triplet_model = Model(inputs, outputs)
 triplet_model.add_loss(K.mean(triplet_loss(outputs)))
 return embedding_model, triplet_model
import matplotlib.pyplot as plt
import os<load_from_csv>
data = pd.read_csv(path_csv)
 train, test = train_test_split(data, train_size=0.7, random_state=1337)
 file_id_mapping_train = {k: v for k, v in zip(train.Image.values, train.Id.values)}
file_id_mapping_test = {k: v for k, v in zip(test.Image.values, test.Id.values)}
gen_tr = gen(sample_gen(file_id_mapping_train))
gen_te = gen(sample_gen(file_id_mapping_test))
checkpoint = ModelCheckpoint(path_model, monitor='loss', verbose=1, save_best_only=True, mode='min')
 early = EarlyStopping(monitor="val_loss", mode="min", patience=2)
 callbacks_list = [checkpoint, early]
data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/train.csv',index_col='Date',parse_dates=True )<categorify>
embedding_model, triplet_model = GetModel()
data = data.fillna(value='empty' )<feature_engineering>
for i, layer in enumerate(embedding_model.layers):
print(i, layer.name, layer.trainable )
data['state_with_country'] = data['Province_State'] +'_'+ data['Country_Region']<drop_column>
for layer in embedding_model.layers[175:]:
layer.trainable = True
for layer in embedding_model.layers[:175]:
layer.trainable = False
data= data.drop(labels=['Province_State','Country_Region','Id'],axis=1 )<define_variables>
triplet_model.compile(loss=None, optimizer=Adam(0.01))
history = triplet_model.fit_generator(gen_tr,
validation_data=gen_te,
epochs=4,
verbose=1,
workers=4,
steps_per_epoch=200,
validation_steps=20 )
state_i_data =[]
for i in state_with_country_name:
state_i_data.append(i )<drop_column>
for layer in embedding_model.layers[150:]:
layer.trainable = True
for layer in embedding_model.layers[:150]:
layer.trainable = False
triplet_model.compile(loss=None, optimizer=Adam(0.0001))
history = triplet_model.fit_generator(gen_tr,
validation_data=gen_te,
epochs=3,
verbose=1,
workers=4,
steps_per_epoch=70,
validation_steps=30 )
for i,j in enumerate(state_with_country_name):
state_i_data[i] = data[data['state_with_country'] == j ]
state_i_data[i] = state_i_data[i].drop(['state_with_country'],axis=1)
<load_from_csv>
def data_generator(fpaths, batch=16):
i = 0
imgs = []
fnames = []
for path in fpaths:
if i == 0:
imgs = []
fnames = []
i += 1
img = read_and_resize(path)
 imgs.append(img)
 fnames.append(os.path.basename(path))
if i == batch:
i = 0
imgs = np.array(imgs)
 yield fnames, imgs
if i != 0:
imgs = np.array(imgs)
 yield fnames, imgs
raise StopIteration()
prev_sub = pd.read_csv('/kaggle/input/rnn-forecasting-data/rnn/test.csv')
 test_whole = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv',index_col='Date',parse_dates=True)
 test_whole = test_whole.fillna(value='empty')
<feature_engineering>
data = pd.read_csv(path_csv)
 file_id_mapping = {k: v for k, v in zip(data.Image.values, data.Id.values)}
train_files = glob.glob(join(path_train, '*.jpg'))
test_files = glob.glob(join(path_test, '*.jpg'))
test_whole['state_with_country'] = test_whole['Province_State'] +'_'+ test_whole['Country_Region']
test_whole.head()<feature_engineering>
train_preds = []
train_file_names = []
for fnames, imgs in tqdm(data_generator(train_files, batch=32)) :
predicts = embedding_model.predict(imgs)
 predicts = predicts.tolist()
train_preds += predicts
train_file_names += fnames
train_preds = np.array(train_preds )
prev_sub.index = test_whole.index<prepare_output>
test_preds = []
test_file_names = []
for fnames, imgs in tqdm(data_generator(test_files, batch=32)) :
predicts = embedding_model.predict(imgs)
 predicts = predicts.tolist()
test_preds += predicts
test_file_names += fnames
test_preds = np.array(test_preds )
prev_sub['state_with_country'] = test_whole['state_with_country']<define_variables>
neigh = NearestNeighbors(n_neighbors=6)
 neigh.fit(train_preds )
forecast = []
for i in state_with_country_name:
forecast.append(i )<drop_column>
preds_str = []
for filepath, distance, neighbour_ in zip(test_file_names, distances_test, neighbors_test):
sample_result = []
sample_classes = []
for d, n in zip(distance, neighbour_):
train_file = train_files[n].split(os.sep)[-1]
class_train = file_id_mapping[train_file]
sample_classes.append(class_train)
 sample_result.append(( class_train, d))
if "new_whale" not in sample_classes:
sample_result.append(( "new_whale", 0.1))
sample_result.sort(key=lambda x: x[1])
 sample_result = sample_result[:5]
preds_str.append(" ".join([x[0] for x in sample_result]))
for i,j in enumerate(state_with_country_name):
forecast[i] = prev_sub[prev_sub['state_with_country'] == j ]
forecast[i] = forecast[i].iloc[13:21]
forecast[i] = forecast[i].drop(['ForecastId','state_with_country'],axis=1)
 state_i_data[i] = state_i_data[i].append(forecast[i] )<load_from_csv>
df = pd.DataFrame(preds_str, columns=["Id"])
 df['Image'] = [x.split(os.sep)[-1] for x in test_file_names]
df.to_csv("sub.csv", index=False )