Kaggle competition: Humpback Whale Identification
<SOS> metric: MAP@{K} Kaggle data source: humpback-whale-identification<feature_engineering>
!pip install fastai==0.7.0 --no-deps
!pip install torch==0.4.1 torchvision==0.2.1
!pip install torchtext==0.2.3
train_df = reduce_mem_usage(train_df, use_float16=True)
 building_meta_df = reduce_mem_usage(building_meta_df, use_float16=True)
 weather_train_df = reduce_mem_usage(weather_train_df, use_float16=True )<define_variables>
MODEL_NAME = 'Densenet201'
TRAIN = '.. /input/train/'
TEST = '.. /input/test/'
LABELS = '.. /input/train.csv'
SAMPLE_SUB = '.. /input/sample_submission.csv'
arch = resnet50
num_workers = 8
category_cols = ['building_id', 'site_id', 'primary_use']
feature_cols = ['square_feet', 'year_built'] + [
'hour', 'weekend',
'building_median'] + [
'air_temperature', 'cloud_coverage',
'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
'wind_direction', 'wind_speed',] 
<prepare_x_and_y>
df = pd.read_csv(LABELS ).set_index('Image')
 new_whale_df = df[df.Id == "new_whale"]
train_df = df[~(df.Id == "new_whale")]
unique_labels = np.unique(train_df.Id.values)
 labels_dict = dict()
labels_list = []
for i in range(len(unique_labels)) :
labels_dict[unique_labels[i]] = i
labels_list.append(unique_labels[i])
 print("Number of classes: {}".format(len(unique_labels)))
 train_df.Id = train_df.Id.apply(lambda x: labels_dict[x])
 train_labels = np.asarray(train_df.Id.values)
 test_names = [f for f in os.listdir(TEST)]
def create_X_y(train_df, target_meter):
target_train_df = train_df[train_df['meter'] == target_meter]
target_train_df = target_train_df.merge(building_meta_df, on='building_id', how='left')
 target_train_df = target_train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 X_train = target_train_df[feature_cols + category_cols]
y_train = target_train_df['meter_reading_log1p'].values
del target_train_df
return X_train, y_train<train_model>
train_df['image_name'] = train_df.index
rs = np.random.RandomState(42)
 perm = rs.permutation(len(train_df))
tr_n = train_df['image_name'].values
val_n = train_df['image_name'].values[perm][:1000]
print('Train/val:', len(tr_n), len(val_n))
print('Train classes', len(train_df.loc[tr_n].Id.unique()))
print('Val classes', len(train_df.loc[val_n].Id.unique()))
def fit_lgbm(train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500, lr=0.1, bf=0.1):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {'num_leaves': 31,
'objective': 'regression',
'learning_rate': lr,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": bf,
"feature_fraction": 0.9,
"metric": metric,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 20
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/mae': model.best_score['training']['l2'],
'valid/mae': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log<choose_model_class>
class HWIDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.train_df = train_df
super().__init__(fnames, transform, path)
 def get_x(self, i):
img = open_image(os.path.join(self.path, self.fnames[i]))
img = cv2.resize(img,(self.sz, self.sz))
return img
def get_y(self, i):
if(self.path == TEST): return 0
return self.train_df.loc[self.fnames[i]]['Id']
def get_c(self):
return len(unique_labels )
seed = 666
shuffle = False
kf = KFold(n_splits=folds, shuffle=shuffle, random_state=seed)
 oof_total = 0<split>
class RandomLighting(Transform):
def __init__(self, b, c, tfm_y=TfmType.NO):
super().__init__(tfm_y)
 self.b, self.c = b, c
def set_state(self):
self.store.b_rand = rand0(self.b)
 self.store.c_rand = rand0(self.c)
 def do_transform(self, x, is_y):
if is_y and self.tfm_y != TfmType.PIXEL: return x
b = self.store.b_rand
c = self.store.c_rand
c = -1 /(c - 1)if c < 0 else c + 1
x = lighting(x, b, c)
 return x
def get_data(sz, batch_size):

 aug_tfms = [RandomRotateZoom(deg=20, zoom=2, stretch=1),
RandomLighting(0.2, 0.2, tfm_y=TfmType.NO),
RandomBlur(blur_strengths=3,tfm_y=TfmType.NO),
RandomFlip(tfm_y=TfmType.NO)]
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.NO,
aug_tfms=aug_tfms)
 ds = ImageData.get_ds(HWIDataset,(tr_n[:-(len(tr_n)% batch_size)], TRAIN), 
(val_n, TRAIN), tfms, test=(test_names, TEST))
md = ImageData("./", ds, batch_size, num_workers=num_workers, classes=None)
 return md
target_meter = 0
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models0 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.7)
 y_valid_pred_total[valid_idx] = y_pred_valid
models0.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof0 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof0 * len(y_train)
 del X_train, y_train
gc.collect()<split>
image_size = 224
batch_size = 48
md = get_data(image_size, batch_size)
 extra_fc_layers_size = []
learn = ConvLearner.pretrained(arch, md, xtra_fc=extra_fc_layers_size)
 learn.opt_fn = optim.Adam
target_meter = 1
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models1 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=num_rounds,
lr=0.05, bf=0.5)
 y_valid_pred_total[valid_idx] = y_pred_valid
models1.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof1 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof1 * len(y_train)
 del X_train, y_train
gc.collect()<split>
print('Number of layer groups:', len(learn.get_layer_groups()), '\t(first 2 groups is pretrained backbone)')
 print('This is our extra thin on top of the backbone Resnet50 architecture:')
 learn.get_layer_groups() [2]
target_meter = 2
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models2 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.8)
 y_valid_pred_total[valid_idx] = y_pred_valid
models2.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof2 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof2 * len(y_train)
 del X_train, y_train
gc.collect()<split>
base_lr = 5e-4
fc_lr = 5e-3
lrs = [base_lr, base_lr, fc_lr]
learn.fit(lrs=lrs, n_cycle=2, cycle_len=None)
 learn.unfreeze()
learn.fit(lrs, n_cycle=3, cycle_len=1, cycle_mult=2)
 learn.save('weights' )
target_meter = 3
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models3 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=num_rounds,
lr=0.03, bf=0.9)
 y_valid_pred_total[valid_idx] = y_pred_valid
models3.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof3 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof3 * len(y_train)
 del X_train, y_train
gc.collect()<define_variables>
image_size = 448
batch_size = 24
md = get_data(image_size, batch_size)
 learn.set_data(md )
oof_total = oof_total / len(train_df )<drop_column>
base_lr = 1e-5
fc_lr = 1e-3
lrs = [base_lr, base_lr, fc_lr]
learn.fit(lrs, n_cycle=6, cycle_len=1)
 learn.save('weights_v2' )
del train_df, weather_train_df, building_meta_df
gc.collect()<feature_engineering>
best_th = 0.38
preds_t,y_t = learn.TTA(is_test=True,n_aug=8)
 preds_t = np.stack(preds_t, axis=-1)
 preds_t = np.exp(preds_t)
 preds_t = preds_t.mean(axis=-1)
 preds_t = np.concatenate([np.zeros(( preds_t.shape[0],1)) +best_th, preds_t],axis=1)
 np.save('preds_dn201.npy', preds_t )
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather')
 set_localtime(weather_test_df)
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 test_df['building_median'] = test_df['building_id'].map(building_median)
 print('preprocessing weather...')
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 print('reduce mem usage...')
 test_df = reduce_mem_usage(test_df, use_float16=True)
 weather_test_df = reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()<load_from_disk>
sample_df = pd.read_csv(SAMPLE_SUB)
 sample_list = list(sample_df.Image)
 labels_list = ["new_whale"]+labels_list
pred_list = [[labels_list[i] for i in p.argsort() [-5:][::-1]] for p in preds_t]
pred_dic = dict(( key, value)for(key, value)in zip(learn.data.test_ds.fnames,pred_list))
pred_list_cor = [' '.join(pred_dic[id])for id in sample_list]
df = pd.DataFrame({'Image':sample_list,'Id': pred_list_cor})
 df.to_csv('sub_{}.csv'.format(MODEL_NAME), header=True, index=False)
 df.head()