Kaggle competition: Humpback Whale Identification
<SOS> metric: MAP@{K} Kaggle data source: humpback-whale-identification<feature_engineering>
from fastai.conv_learner import *
from fastai.dataset import *
from tqdm import tqdm
import pandas as pd
import numpy as np
import os
from sklearn.model_selection import train_test_split, StratifiedShuffleSplit
import matplotlib.pyplot as plt
import math
if use_ucf:
leak0_df = leak0_df[leak0_df.timestamp.dt.year.isin(ucf_year)]
leak1_df = leak1_df[leak1_df.timestamp.dt.year.isin(ucf_year)]
leak2_df = leak2_df[leak2_df.timestamp.dt.year.isin(ucf_year)]
train_df = pd.concat([train_df, leak0_df, leak1_df, leak2_df])
 train_df.reset_index(inplace=True)
 weather_train_df = pd.concat([weather_train_df, weather_test_df])
 weather_train_df.reset_index(inplace=True )<drop_column>
MODEL_PATH = 'Resnet18_v1'
TRAIN = '.. /input/train/'
TEST = '.. /input/test/'
LABELS = '.. /input/train.csv'
SAMPLE_SUB = '.. /input/sample_submission.csv'
del weather_test_df, leak0_df, leak1_df, leak2_df
gc.collect()<feature_engineering>
arch = resnet34
nw = 4
train_df['date'] = train_df['timestamp'].dt.date
train_df['meter_reading_log1p'] = np.log1p(train_df['meter_reading'] )<filter>
df = pd.read_csv(LABELS ).set_index('Image')
 new_whale_df = df[df.Id == "new_whale"]
train_df = df[~(df.Id == "new_whale")]
unique_labels = np.unique(train_df.Id.values)
 labels_dict = dict()
labels_list = []
for i in range(len(unique_labels)) :
labels_dict[unique_labels[i]] = i
labels_list.append(unique_labels[i])
 print("Number of classes: {}".format(len(unique_labels)))
 train_names = train_df.index.values
train_df.Id = train_df.Id.apply(lambda x: labels_dict[x])
 train_labels = np.asarray(train_df.Id.values)
 test_names = [f for f in os.listdir(TEST)]
building_meta_df[building_meta_df.site_id == 0]<filter>
dup = []
for idx,row in train_df.iterrows() :
if labels_count[row['Id']] < 5:
dup.extend([idx]*math.ceil(( 5 - labels_count[row['Id']])/labels_count[row['Id']]))
train_names = np.concatenate([train_names, dup])
 train_names = train_names[np.random.RandomState(seed=42 ).permutation(train_names.shape[0])]
len(train_names )
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )<feature_engineering>
sss = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42069)
 for train_idx, val_idx in sss.split(train_names, np.zeros(train_names.shape)) :
tr_n, val_n = train_names[train_idx], train_names[val_idx]
print(len(tr_n), len(val_n))
zone_dict={0:4,1:0,2:7,3:4,4:7,5:0,6:4,7:4,8:4,9:5,10:7,11:4,12:0,13:5,14:4,15:4}
def set_localtime(df):
for sid, zone in zone_dict.items() :
sids = df.site_id == sid
df.loc[sids, 'timestamp'] = df[sids].timestamp - pd.offsets.Hour(zone )<feature_engineering>
avg_width = 0
avg_height = 0
for fn in os.listdir(TRAIN)[:1000]:
img = cv2.imread(os.path.join(TRAIN,fn))
avg_width += img.shape[1]
avg_height += img.shape[0]
avg_width //= 1000
avg_height //= 1000
print(avg_width, avg_height )
def preprocess(df):
df["hour"] = df["timestamp"].dt.hour
df["weekend"] = df["timestamp"].dt.weekday
df["month"] = df["timestamp"].dt.month
df["dayofweek"] = df["timestamp"].dt.dayofweek 
<categorify>
class HWIDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.train_df = train_df
super().__init__(fnames, transform, path)
 def get_x(self, i):
img = open_image(os.path.join(self.path, self.fnames[i]))
img = cv2.resize(img,(self.sz, self.sz))
return img
def get_y(self, i):
if(self.path == TEST): return 0
return self.train_df.loc[self.fnames[i]]['Id']
def get_c(self):
return len(unique_labels)
preprocess(train_df )<sort_values>
class RandomLighting(Transform):
def __init__(self, b, c, tfm_y=TfmType.NO):
super().__init__(tfm_y)
 self.b, self.c = b, c
def set_state(self):
self.store.b_rand = rand0(self.b)
 self.store.c_rand = rand0(self.c)
 def do_transform(self, x, is_y):
if is_y and self.tfm_y != TfmType.PIXEL: return x
b = self.store.b_rand
c = self.store.c_rand
c = -1 /(c - 1)if c < 0 else c + 1
x = lighting(x, b, c)
 return x
def get_data(sz, bs, test_names=test_names, test_dir=TEST):
aug_tfms = [RandomRotateZoom(deg=20, zoom=2, stretch=1),
RandomLighting(0.05, 0.05, tfm_y=TfmType.NO),
RandomBlur(blur_strengths=3,tfm_y=TfmType.NO),
RandomFlip(tfm_y=TfmType.NO)]
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.NO,
aug_tfms=aug_tfms)
 ds = ImageData.get_ds(HWIDataset,(tr_n[:-(len(tr_n)% bs)], TRAIN), 
(val_n, TRAIN), tfms, test=(test_names, test_dir))
md = ImageData("./", ds, bs, num_workers=nw, classes=None)
 return md
if use_ucf and use_sort:
train_df = train_df.sort_values('month')
 train_df = train_df.reset_index()<categorify>
batch_size = 64
md = get_data(384, batch_size)
 learn = ConvLearner.pretrained(arch, md)
 learn.opt_fn = optim.Adam
df_group = train_df.groupby('building_id')['meter_reading_log1p']
building_median = df_group.median().astype(np.float16)
 train_df['building_median'] = train_df['building_id'].map(building_median)
 del df_group<count_missing_values>
learn.fit(lr, 2, cycle_len=3)
 learn.unfreeze()
lrs = np.array([lr/10, lr/20, lr/40])
 learn.fit(lrs, 4, cycle_len=4, use_clr=(20, 16))
learn.save(MODEL_PATH )
weather_train_df.isna().sum()<groupby>
preds_v,y_v = learn.TTA(is_test=False,n_aug=2)
 preds_v = np.stack(preds_v, axis=-1)
 preds_v = np.exp(preds_v)
 preds_v = preds_v.mean(axis=-1)
 y_v += 1
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )<groupby>
TEST=TRAIN
total_new_whale = len(new_whale_df.index.values)
 md = get_data(384, batch_size, test_names=new_whale_df.index.values[:int(total_new_whale*0.2)], test_dir=TRAIN)
 learn.set_data(md)
 preds_w,y_w = learn.TTA(is_test=True,n_aug=2)
 preds_w = np.stack(preds_w, axis=-1)
 preds_w = np.exp(preds_w)
 preds_w = preds_w.mean(axis=-1 )
weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))<groupby>
y = np.concatenate([y_v,y_w])
 preds = np.concatenate([preds_v, preds_w],axis=0)
 preds = np.concatenate([np.zeros(( preds.shape[0],1)) , preds],axis=1 )
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )<data_type_conversions>
def map5(X, y):
score = 0
for i in range(X.shape[0]):
pred = X[i].argsort() [-5:][::-1]
for j in range(pred.shape[0]):
if pred[j] == y[i]:
score +=(5 - j)/5
break
return score/X.shape[0]
best_th = 0
best_score = 0
for th in np.arange(0.1, 0.801, 0.01):
preds[:,0] = th
score = map5(preds, y)
 if score > best_score:
best_score = score
best_th = th
print("Threshold = {:.3f}, MAP5 = {:.3f}".format(th,score))
def add_lag_feature(weather_df, window=3):
group_df = weather_df.groupby('site_id')
 cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']
rolled = group_df[cols].rolling(window=window, min_periods=0)
 lag_mean = rolled.mean().reset_index().astype(np.float16)
 lag_max = rolled.max().reset_index().astype(np.float16)
 lag_min = rolled.min().reset_index().astype(np.float16)
 lag_std = rolled.std().reset_index().astype(np.float16)
 for col in cols:
weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]
weather_df[f'{col}_max_lag{window}'] = lag_max[col]
weather_df[f'{col}_min_lag{window}'] = lag_min[col]
weather_df[f'{col}_std_lag{window}'] = lag_std[col]<feature_engineering>
TEST = '.. /input/test/'
md = get_data(384, batch_size, test_names=test_names, test_dir=TEST)
 learn.set_data(md)
 preds_t,y_t = learn.TTA(is_test=True,n_aug=8)
 preds_t = np.stack(preds_t, axis=-1)
 preds_t = np.exp(preds_t)
 preds_t = preds_t.mean(axis=-1)
 preds_t = np.concatenate([np.zeros(( preds_t.shape[0],1)) +best_th, preds_t],axis=1 )
<categorify><EOS> 
sample_df = pd.read_csv(SAMPLE_SUB)
 sample_list = list(sample_df.Image)
 labels_list = ["new_whale"]+labels_list
pred_list = [[labels_list[i] for i in p.argsort() [-5:][::-1]] for p in preds_t]
pred_dic = dict(( key, value)for(key, value)in zip(learn.data.test_ds.fnames,pred_list))
pred_list_cor = [' '.join(pred_dic[id])for id in sample_list]
df = pd.DataFrame({'Image':sample_list,'Id': pred_list_cor})
 df.to_csv('submission.csv'.format(MODEL_PATH), header=True, index=False)
 df.head()