Kaggle competition: Humpback Whale Identification
<SOS> metric: MAP@{K} Kaggle data source: humpback-whale-identification<compute_test_metric>
!pip install fastai==0.7.0 --no-deps
!pip install torch==0.4.1 torchvision==0.2.1
!pip install torchtext==0.2.3
def loss_fn(start_logits, end_logits, start_positions, end_positions):
ce_loss = nn.CrossEntropyLoss()
start_loss = ce_loss(start_logits, start_positions)
 end_loss = ce_loss(end_logits, end_positions)
 total_loss = start_loss + end_loss
return total_loss<statistical_test>
MODEL_NAME = 'Densenet201'
TRAIN = '.. /input/train/'
TEST = '.. /input/test/'
LABELS = '.. /input/train.csv'
SAMPLE_SUB = '.. /input/sample_submission.csv'
arch = dn201
num_workers = 8
def get_selected_text(text, start_idx, end_idx, offsets):
selected_text = ""
for ix in range(start_idx, end_idx + 1):
selected_text += text[offsets[ix][0]: offsets[ix][1]]
if(ix + 1)< len(offsets)and offsets[ix][1] < offsets[ix + 1][0]:
selected_text += " "
return selected_text
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
def compute_jaccard_score(text, start_idx, end_idx, start_logits, end_logits, offsets):
start_pred = np.argmax(start_logits)
 end_pred = np.argmax(end_logits)
 if start_pred > end_pred:
pred = text
else:
pred = get_selected_text(text, start_pred, end_pred, offsets)
 true = get_selected_text(text, start_idx, end_idx, offsets)
 return jaccard(true, pred )<train_model>
df = pd.read_csv(LABELS ).set_index('Image')
 new_whale_df = df[df.Id == "new_whale"]
train_df = df[~(df.Id == "new_whale")]
unique_labels = np.unique(train_df.Id.values)
 labels_dict = dict()
labels_list = []
for i in range(len(unique_labels)) :
labels_dict[unique_labels[i]] = i
labels_list.append(unique_labels[i])
 print("Number of classes: {}".format(len(unique_labels)))
 train_df.Id = train_df.Id.apply(lambda x: labels_dict[x])
 train_labels = np.asarray(train_df.Id.values)
 test_names = [f for f in os.listdir(TEST)]
def train_model(model, dataloaders_dict, criterion, optimizer, num_epochs):
if e_cuda:
model.cuda()
for epoch in range(num_epochs):
for phase in ['train', 'val']:
if phase == 'train':
model.train()
else:
model.eval()
epoch_loss = 0.0
epoch_jaccard = 0.0
for data in(dataloaders_dict[phase]):
ids = data['ids']
masks = data['masks']
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_idx = data['start_idx']
end_idx = data['end_idx']
if e_cuda:
ids = ids.cuda()
masks = masks.cuda()
start_idx = start_idx.cuda()
end_idx = end_idx.cuda()
optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
start_logits, end_logits = model(ids, masks)
 loss = criterion(start_logits, end_logits, start_idx, end_idx)
 if phase == 'train':
loss.backward()
optimizer.step()
epoch_loss += loss.item() * len(ids)
 start_idx = start_idx.cpu().detach().numpy()
end_idx = end_idx.cpu().detach().numpy()
start_logits = torch.softmax(start_logits, dim=1 ).cpu().detach().numpy()
end_logits = torch.softmax(end_logits, dim=1 ).cpu().detach().numpy()
for i in range(len(ids)) :
jaccard_score = compute_jaccard_score(
 tweet[i],
start_idx[i],
end_idx[i],
start_logits[i],
end_logits[i],
offsets[i])
 epoch_jaccard += jaccard_score
epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)
 epoch_jaccard = epoch_jaccard / len(dataloaders_dict[phase].dataset)
 print('| {}/{} | {:^5} | {:.4f} | {:.4f} |'.format(
 epoch + 1, num_epochs, phase, epoch_loss, epoch_jaccard)) 
<load_pretrained>
train_df['image_name'] = train_df.index
rs = np.random.RandomState(42)
 perm = rs.permutation(len(train_df))
tr_n = train_df['image_name'].values
val_n = train_df['image_name'].values[perm][:1000]
print('Train/val:', len(tr_n), len(val_n))
print('Train classes', len(train_df.loc[tr_n].Id.unique()))
print('Val classes', len(train_df.loc[val_n].Id.unique()))
def init_logits(test_loader):
s_logits = {}
e_logits = {}
for i, data in enumerate(test_loader):
s_logits[i] = []
e_logits[i] = []
return s_logits, e_logits<load_pretrained>
class HWIDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.train_df = train_df
super().__init__(fnames, transform, path)
 def get_x(self, i):
img = open_image(os.path.join(self.path, self.fnames[i]))
img = cv2.resize(img,(self.sz, self.sz))
return img
def get_y(self, i):
if(self.path == TEST): return 0
return self.train_df.loc[self.fnames[i]]['Id']
def get_c(self):
return len(unique_labels )
class TweetModel(nn.Module):
def __init__(self, p_drop, l_std, m_size):
super(TweetModel, self ).__init__()
config = RobertaConfig.from_pretrained(
 config_roberta_config, output_hidden_states=True)
 self.roberta = RobertaModel.from_pretrained(
 config_roberta_model, config=config)
 self.dropout = nn.Dropout(p_drop)
 self.conv = nn.Conv1d(config.hidden_size, m_size , kernel_size = 2)
 self.conv2 = nn.Conv1d(config.hidden_size, m_size, kernel_size = 2)
 self.fc = nn.Linear(m_size, 1)
 nn.init.normal_(self.fc.weight, std=l_std)
 nn.init.normal_(self.fc.bias, 0)
 self.fc2 = nn.Linear(m_size, 1)
 nn.init.normal_(self.fc2.weight, std=l_std)
 nn.init.normal_(self.fc2.bias, 0)
 def forward(self, input_ids, attention_mask):
_, _, hs = self.roberta(input_ids, attention_mask)
 x = torch.stack([hs[-1], hs[-2], hs[-3]])
 x = torch.mean(x, 0)
 x = self.dropout(x ).transpose(1,2)
 x1 = F.leaky_relu(self.conv(x)).transpose(1,2)
 x2 = F.leaky_relu(self.conv2(x)).transpose(1,2)
 start_logits = self.fc(x1 ).squeeze(-1)
 end_logits = self.fc2(x2 ).squeeze(-1)
 return start_logits, end_logits<define_variables>
class RandomLighting(Transform):
def __init__(self, b, c, tfm_y=TfmType.NO):
super().__init__(tfm_y)
 self.b, self.c = b, c
def set_state(self):
self.store.b_rand = rand0(self.b)
 self.store.c_rand = rand0(self.c)
 def do_transform(self, x, is_y):
if is_y and self.tfm_y != TfmType.PIXEL: return x
b = self.store.b_rand
c = self.store.c_rand
c = -1 /(c - 1)if c < 0 else c + 1
x = lighting(x, b, c)
 return x
def get_data(sz, batch_size):

 aug_tfms = [RandomRotateZoom(deg=20, zoom=2, stretch=1),
RandomLighting(0.2, 0.2, tfm_y=TfmType.NO),
RandomBlur(blur_strengths=3,tfm_y=TfmType.NO),
RandomFlip(tfm_y=TfmType.NO)]
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.NO,
aug_tfms=aug_tfms)
 ds = ImageData.get_ds(HWIDataset,(tr_n[:-(len(tr_n)% batch_size)], TRAIN), 
(val_n, TRAIN), tfms, test=(test_names, TEST))
md = ImageData("./", ds, batch_size, num_workers=num_workers, classes=None)
 return md
seed = 93
num_epochs = 3
batch_size = 32
n_splits = 10
p_drop = 0.1
learning_rate=3e-5
l_std=0.03
m_size=128
print(f'| param | |')
 print(f'|--------|--------|')
 print(f'| seed | {seed} |')
 print(f'| epochs | {num_epochs} |')
 print(f'| batch | {batch_size} |')
 print(f'| splits | {n_splits} |')
 print(f'| p_drop | {p_drop} |')
 print(f'| lr | {learning_rate} |')
 print(f'| std | {l_std} |')
 print(f'| m_size | {m_size} |')
 skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed)
 train_df = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train_df['text'] = train_df['text'].astype(str)
 train_df['selected_text'] = train_df['selected_text'].astype(str)
 test_df = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test_df['text'] = test_df['text'].astype(str)
 test_loader = get_test_loader(test_df, batch_size)
 s_logits, e_logits = init_logits(test_loader)
 for fold,(train_idx, val_idx)in enumerate(skf.split(train_df, train_df.sentiment)) :
print_and_log(f'Fold: {fold+1}')
 print('| Epoch | | Loss | Jaccard |')
 print('|-------|-------|--------|---------|')
 model = TweetModel(p_drop, l_std, m_size)
 optimizer = optim.AdamW(model.parameters() , lr=learning_rate, betas=(0.9, 0.999))
criterion = loss_fn
dataloaders_dict = get_train_val_loaders(train_df, train_idx, val_idx, batch_size)
 train_model(
 model,
dataloaders_dict,
criterion,
optimizer,
num_epochs)
 for i, data in enumerate(test_loader):
ids = data['ids']
masks = data['masks']
if e_cuda:
ids = ids.cuda()
masks = masks.cuda()
with torch.no_grad() :
output = model(ids, masks)
 s_logits[i].append(torch.softmax(output[0], dim=1 ).cpu().detach().numpy())
 e_logits[i].append(torch.softmax(output[1], dim=1 ).cpu().detach().numpy() )<save_model>
image_size = 224
batch_size = 48
md = get_data(image_size, batch_size)
 extra_fc_layers_size = []
learn = ConvLearner.pretrained(arch, md, xtra_fc=extra_fc_layers_size)
 learn.opt_fn = optim.Adam
np.save('s_logits.npy', s_logits)
 np.save('e_logits.npy', e_logits )<feature_engineering>
print('Number of layer groups:', len(learn.get_layer_groups()), '\t(first 2 groups is pretrained backbone)')
 print('This is our extra thin on top of the backbone Resnet50 architecture:')
 learn.get_layer_groups() [2]
predictions = []
for i, data in enumerate(test_loader):
ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_logits = np.mean(s_logits[i], axis=0)
 end_logits = np.mean(e_logits[i], axis=0)
 for i in range(len(ids)) :
start_pred = np.argmax(start_logits[i])
 end_pred = np.argmax(end_logits[i])
 if start_pred > end_pred:
pred = tweet[i]
else:
pred = get_selected_text(tweet[i], start_pred, end_pred, offsets[i])
 predictions.append(pred )<save_to_csv>
base_lr = 5e-4
fc_lr = 5e-3
lrs = [base_lr, base_lr, fc_lr]
learn.fit(lrs=lrs, n_cycle=2, cycle_len=None)
 learn.unfreeze()
learn.fit(lrs, n_cycle=3, cycle_len=1, cycle_mult=2)
 learn.save('weights' )
sub_df = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 sub_df['selected_text'] = predictions
sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('!!!!', '!')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('.. ', '.')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('...', '.')if len(x.split())==1 else x)
 sub_df.to_csv('submission.csv', index=False)
 sub_df.head()<import_modules>
image_size = 448
batch_size = 24
md = get_data(image_size, batch_size)
 learn.set_data(md )
print('TF version',tf.__version__ )<define_variables>
base_lr = 1e-5
fc_lr = 1e-3
lrs = [base_lr, base_lr, fc_lr]
learn.fit(lrs, n_cycle=6, cycle_len=1)
 learn.save('weights_v2' )
MAX_LEN = 96
PATH = '.. /input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=PATH+'vocab-roberta-base.json',
merges_file=PATH+'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True 
)
 sentiment_id = {'positive': 1313, 'negative': 2430, 'neutral': 7974}
train = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv' ).fillna('')
 train.head()<define_variables>
best_th = 0.38
preds_t,y_t = learn.TTA(is_test=True,n_aug=8)
 preds_t = np.stack(preds_t, axis=-1)
 preds_t = np.exp(preds_t)
 preds_t = preds_t.mean(axis=-1)
 preds_t = np.concatenate([np.zeros(( preds_t.shape[0],1)) +best_th, preds_t],axis=1)
 np.save('preds_dn201.npy', preds_t )
ct = train.shape[0]
input_ids = np.ones(( ct,MAX_LEN),dtype='int32')
 attention_mask = np.zeros(( ct,MAX_LEN),dtype='int32')
 token_type_ids = np.zeros(( ct,MAX_LEN),dtype='int32')
 start_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 end_tokens = np.zeros(( ct,MAX_LEN),dtype='int32')
 for k in range(train.shape[0]):
text1 = " "+" ".join(train.loc[k,'text'].split())
 text2 = " ".join(train.loc[k,'selected_text'].split())
 idx = text1.find(text2)
 chars = np.zeros(( len(text1)))
 chars[idx:idx+len(text2)]=1
if text1[idx-1]==' ': chars[idx-1] = 1
enc = tokenizer.encode(text1)
 offsets = []; idx=0
for t in enc.ids:
w = tokenizer.decode([t])
 offsets.append(( idx,idx+len(w)))
 idx += len(w)
 toks = []
for i,(a,b)in enumerate(offsets):
sm = np.sum(chars[a:b])
 if sm>0: toks.append(i)
 s_tok = sentiment_id[train.loc[k,'sentiment']]
input_ids[k,:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]
attention_mask[k,:len(enc.ids)+5] = 1
if len(toks)>0:
start_tokens[k,toks[0]+1] = 1
end_tokens[k,toks[-1]+1] = 1<feature_engineering>
sample_df = pd.read_csv(SAMPLE_SUB)
 sample_list = list(sample_df.Image)
 labels_list = ["new_whale"]+labels_list
pred_list = [[labels_list[i] for i in p.argsort() [-5:][::-1]] for p in preds_t]
pred_dic = dict(( key, value)for(key, value)in zip(learn.data.test_ds.fnames,pred_list))
pred_list_cor = [' '.join(pred_dic[id])for id in sample_list]
df = pd.DataFrame({'Image':sample_list,'Id': pred_list_cor})
 df.to_csv('sub_{}.csv'.format(MODEL_NAME), header=True, index=False)
 df.head()