Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<import_modules>
data = pd.read_csv('/kaggle/input/train_labels.csv')
 train_path = '/kaggle/input/train/'
test_path = '/kaggle/input/test/'
data['label'].value_counts()
from dataset10 import TrainDataset, MyCollator<load_from_csv>
def readImage(path):
bgr_img = cv2.imread(path)
 b,g,r = cv2.split(bgr_img)
 rgb_img = cv2.merge([r,g,b])
 return rgb_img
test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv' )<load_pretrained>
train_df = data.set_index('id')
 train_names = train_df.index.values
train_labels = np.asarray(train_df['label'].values)
 tr_n, tr_idx, val_n, val_idx = train_test_split(train_names, range(len(train_names)) , test_size=0.2, stratify=train_labels, random_state=123 )
tokenizer = AutoTokenizer.from_pretrained('.. /input/roberta-base/', do_lower_case=False )<set_options>
arch = squeezenet1_1
BATCH_SIZE = 128
sz = CROP_SIZE
MODEL_PATH = str(arch ).split() [1]
class Args:
post = True
tokenizer = tokenizer
offset = 4
batch_size = 32
workers = 1
args = Args()<create_dataframe>
train_dict = {'name': train_path + train_names, 'label': train_labels}
df = pd.DataFrame(data=train_dict)
 test_names = []
for f in os.listdir(test_path):
test_names.append(test_path + f)
 df_test = pd.DataFrame(np.asarray(test_names), columns=['name'] )
collator = MyCollator()
test_set = TrainDataset(test, None, tokenizer=tokenizer, mode='test', offset=args.offset)
 test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False, collate_fn=collator,
num_workers=args.workers )<load_pretrained>
class MyImageItemList(ImageList):
def open(self, fn:PathOrStr)->Image:
img = readCroppedImage(fn.replace('/./','' ).replace('//','/'))
return vision.Image(px=pil2tensor(img, np.float32))
class TweetModel(nn.Module):
def __init__(self, pretrain_path=None, dropout=0.2, config=None):
super(TweetModel, self ).__init__()
if config is not None:
self.bert = AutoModel.from_config(config)
 else:
config = AutoConfig.from_pretrained(pretrain_path, output_hidden_states=True)
 self.bert = AutoModel.from_pretrained(
 pretrain_path, cache_dir=None, config=config)
 self.cnn = nn.Conv1d(self.bert.config.hidden_size*3, self.bert.config.hidden_size, 3, padding=1)
 self.gelu = nn.GELU()
self.whole_head = nn.Sequential(OrderedDict([ 
('dropout', nn.Dropout(0.1)) , 
('l1', nn.Linear(self.bert.config.hidden_size*3, 256)) , 
('act1', nn.GELU()), 
('dropout', nn.Dropout(0.1)) , 
('l2', nn.Linear(256, 2))
]))
self.se_head = nn.Linear(self.bert.config.hidden_size, 2)
 self.inst_head = nn.Linear(self.bert.config.hidden_size, 2)
 self.dropout = nn.Dropout(0.1)
 def forward(self, inputs, masks, token_type_ids=None, input_emb=None):
_, pooled_output, hs = self.bert(
 inputs, masks, token_type_ids=token_type_ids, inputs_embeds=input_emb)
 seq_output = torch.cat([hs[-1],hs[-2],hs[-3]], dim=-1)
 avg_output = torch.sum(seq_output*masks.unsqueeze(-1), dim=1, keepdim=False)
 avg_output = avg_output/torch.sum(masks, dim=-1, keepdim=True)
 whole_out = self.whole_head(avg_output)
 seq_output = self.gelu(self.cnn(seq_output.permute(0,2,1)).permute(0,2,1))
se_out = self.se_head(self.dropout(seq_output))
inst_out = self.inst_head(self.dropout(seq_output))
return whole_out, se_out[:, :, 0], se_out[:, :, 1], inst_out<find_best_model_class>
imgDataBunch =(MyImageItemList.from_df(path='/', df=df, suffix='.tif')
.split_by_idx(val_idx)
.label_from_df(cols='label')
.add_test(MyImageItemList.from_df(path='/', df=df_test)) 
.transform(tfms=[[],[]], size=sz)
.databunch(bs=BATCH_SIZE)
.normalize([tensor([0.702447, 0.546243, 0.696453]), tensor([0.238893, 0.282094, 0.216251])])
 )
def predict(model: nn.Module, valid_df, valid_loader, args, progress=False)-> Dict[str, float]:
model.eval()
all_end_pred, all_whole_pred, all_start_pred, all_inst_out = [], [], [], []
if progress:
tq = tqdm.tqdm(total=len(valid_df))
with torch.no_grad() :
for tokens, types, masks, _, _, _, _, _, _, _ in valid_loader:
if progress:
batch_size = tokens.size(0)
 tq.update(batch_size)
 masks = masks.cuda()
tokens = tokens.cuda()
types = types.cuda()
whole_out, start_out, end_out, inst_out = model(tokens, masks, types)
 all_whole_pred.append(torch.softmax(whole_out, dim=-1)[:,1].cpu().numpy())
 inst_out = torch.softmax(inst_out, dim=-1)
 for idx in range(len(start_out)) :
length = torch.sum(masks[idx,:] ).item() -1
all_start_pred.append(torch.softmax(start_out[idx, args.offset:length], axis=-1 ).cpu())
 all_end_pred.append(torch.softmax(end_out[idx, args.offset:length], axis=-1 ).cpu())
 all_inst_out.append(inst_out[idx,:,1].cpu())
 assert all_start_pred[-1].dim() ==1
all_whole_pred = np.concatenate(all_whole_pred)
 if progress:
tq.close()
return all_whole_pred, all_start_pred, all_end_pred, all_inst_out<load_pretrained>
imgDataBunch.show_batch(rows=2, figsize=(4,4))
config = RobertaConfig.from_pretrained('.. /input/roberta-base', output_hidden_states=True)
 model = TweetModel(config=config)
<predict_on_test>
def getLearner() :
return create_cnn(imgDataBunch, arch, pretrained=True, path='.', metrics=accuracy, ps=0.5, callback_fns=ShowGraph)
 learner = getLearner()
all_whole_preds, all_start_preds, all_end_preds, all_inst_preds = [], [], [], []
for fold in range(10):
load_model(model, '.. /input/roberta-v10-10/best-model-%d.pt' % fold)
 model.cuda()
fold_whole_preds, fold_start_preds, fold_end_preds, fold_inst_preds = predict(model, test, test_loader, args, progress=True)
 all_whole_preds.append(fold_whole_preds)
 all_start_preds.append(fold_start_preds)
 all_end_preds.append(fold_end_preds)
 all_inst_preds.append(fold_inst_preds)
 all_whole_preds, all_start_preds, all_end_preds, all_inst_preds = ensemble(all_whole_preds, all_start_preds, all_end_preds, all_inst_preds, test)
 word_preds, inst_word_preds, scores = get_predicts_from_token_logits(all_whole_preds, all_start_preds, all_end_preds, all_inst_preds, test, args)
 start_char_prob, end_char_prob = get_char_prob(all_start_preds, all_end_preds, test, args)
<feature_engineering>
lrs = []
losses = []
wds = []
iter_count = 600
learner.lr_find(wd=1e-6, num_it=iter_count)
 lrs.append(learner.recorder.lrs)
 losses.append(learner.recorder.losses)
 wds.append('1e-6')
 learner = getLearner()
learner.lr_find(wd=1e-4, num_it=iter_count)
 lrs.append(learner.recorder.lrs)
 losses.append(learner.recorder.losses)
 wds.append('1e-4')
 learner = getLearner()
learner.lr_find(wd=1e-2, num_it=iter_count)
 lrs.append(learner.recorder.lrs)
 losses.append(learner.recorder.losses)
 wds.append('1e-2')
 learner = getLearner()
test['start_char_prob'] = start_char_prob
test['end_char_prob'] = end_char_prob<save_to_csv>
max_lr = 2e-2
wd = 1e-4
learner.fit_one_cycle(cyc_len=8, max_lr=max_lr, wd=wd )
test['selected_text'] = word_preds
def f(selected):
return " ".join(set(selected.lower().split()))
test[['textID','selected_text']].to_csv('submission.csv', index=False )<set_options>
learner.save(MODEL_PATH + '_stage1' )
warnings.filterwarnings('ignore' )<set_options>
learner.fit_one_cycle(cyc_len=12, max_lr=slice(4e-5,4e-4))
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed = 42
seed_everything(seed )<load_pretrained>
learner.save(MODEL_PATH + '_stage2' )
class TweetDataset(torch.utils.data.Dataset):
def __init__(self, df, max_len=192):
self.df = df
self.max_len = max_len
self.labeled = 'selected_text' in df
self.tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file='.. /input/roberta-base/vocab.json',
merges_file='.. /input/roberta-base/merges.txt',
lowercase=True,
add_prefix_space=True)
 def __len__(self):
return len(self.df)
 def get_input_data(self, row):
tweet = " " + " ".join(row.text.lower().split())
 encoding = self.tokenizer.encode(tweet)
 sentiment_id = self.tokenizer.encode(row.sentiment ).ids
ids = [0] + sentiment_id + [2, 2] + encoding.ids + [2]
masks=[1, 1, 1, 1] + encoding.attention_mask + [1]
offsets = [(0, 0)] * 4 + encoding.offsets + [(0, 0)]
token_type_ids=[0, 0, 0, 0] + encoding.type_ids + [0]
pad_len = self.max_len - len(ids)
 if pad_len > 0:
ids += [1] * pad_len
masks += [0]*pad_len
offsets += [(0, 0)] * pad_len
token_type_ids += [0]*pad_len
ids = torch.tensor(ids)
 masks = torch.tensor(masks)
 offsets = torch.tensor(offsets)
 token_type_ids = torch.tensor(token_type_ids)
 return ids, masks, tweet, offsets , token_type_ids
def get_target_idx(self, row, tweet, offsets):
selected_text = " " + " ".join(row.selected_text.lower().split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
target_idx = []
for j,(offset1, offset2)in enumerate(offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 start_idx = target_idx[0]
end_idx = target_idx[-1]
return start_idx, end_idx
def __getitem__(self, index):
data = {}
row = self.df.iloc[index]
ids, masks, tweet, offsets, token_type_ids = self.get_input_data(row)
 data['ids'] = ids
data['masks'] = masks
data['tweet'] = tweet
data['offsets'] = offsets
data['token_type_ids'] = token_type_ids
if self.labeled:
start_idx, end_idx = self.get_target_idx(row, tweet, offsets)
 data['start_idx'] = start_idx
data['end_idx'] = end_idx
return data
def get_train_val_loaders(df, train_idx, val_idx, batch_size=8):
train_df = df.iloc[train_idx]
val_df = df.iloc[val_idx]
train_loader = torch.utils.data.DataLoader(
 TweetDataset(train_df),
batch_size=batch_size,
shuffle=True,
num_workers=2,
drop_last=True)
 val_loader = torch.utils.data.DataLoader(
 TweetDataset(val_df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 dataloaders_dict = {"train": train_loader, "val": val_loader}
return dataloaders_dict
def get_test_loader(df, batch_size=32):
loader = torch.utils.data.DataLoader(
 TweetDataset(df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 return loader<load_pretrained>
preds,y, loss = learner.get_preds(with_loss=True)
 acc = accuracy(preds, y)
 print('The accuracy is {0} %.'.format(acc))
class TweetModel(nn.Module):
def __init__(self):
super(TweetModel, self ).__init__()
config = RobertaConfig.from_pretrained(
 '.. /input/roberta-base/config.json', output_hidden_states=True)
 self.roberta = RobertaModel.from_pretrained(
 '.. /input/roberta-base/pytorch_model.bin', config=config)
 self.dropout = nn.Dropout(0.5)
 self.fc = nn.Linear(config.hidden_size, 2)
 nn.init.normal_(self.fc.weight, std=0.02)
 nn.init.normal_(self.fc.bias, 0)
 def forward(self, input_ids, attention_mask):
_, _, hs = self.roberta(input_ids, attention_mask)
 x = torch.stack([hs[-1], hs[-2], hs[-3], hs[-4]])
 x = torch.mean(x, 0)
 x = self.dropout(x)
 x = self.fc(x)
 start_logits, end_logits = x.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<compute_test_metric>
def hooked_backward(m, oneBatch, cat):
with hook_output(m[0])as hook_a:
with hook_output(m[0], grad=True)as hook_g:
preds = m(oneBatch)
 preds[0,int(cat)].backward()
return hook_a,hook_g
def loss_fn(start_logits, end_logits, start_positions, end_positions):
ce_loss = nn.CrossEntropyLoss()
start_loss = ce_loss(start_logits, start_positions)
 end_loss = ce_loss(end_logits, end_positions)
 total_loss = start_loss + end_loss
return total_loss<statistical_test>
probs = np.exp(preds[:,1])
 fpr, tpr, thresholds = roc_curve(y, probs, pos_label=1)
 roc_auc = auc(fpr, tpr)
 print('ROC area is {0}'.format(roc_auc))
def get_selected_text(text, start_idx, end_idx, offsets):
selected_text = ""
for ix in range(start_idx, end_idx + 1):
selected_text += text[offsets[ix][0]: offsets[ix][1]]
if(ix + 1)< len(offsets)and offsets[ix][1] < offsets[ix + 1][0]:
selected_text += " "
return selected_text
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
def compute_jaccard_score(text, start_idx, end_idx, start_logits, end_logits, offsets):
start_pred = np.argmax(start_logits)
 end_pred = np.argmax(end_logits)
 if start_pred > end_pred:
pred = text
else:
pred = get_selected_text(text, start_pred, end_pred, offsets)
 true = get_selected_text(text, start_idx, end_idx, offsets)
 return jaccard(true, pred )<train_model>
learner.load(MODEL_PATH + '_stage2')
 n_aug = 12
preds_n_avg = np.zeros(( len(learner.data.test_ds.items),2))
for n in tqdm_notebook(range(n_aug), 'Running TTA...'):
preds,y = learner.get_preds(ds_type=DatasetType.Test, with_loss=False)
 preds_n_avg = np.sum([preds_n_avg, preds.numpy() ], axis=0)
 preds_n_avg = preds_n_avg / n_aug
def train_model(model, dataloaders_dict, criterion, optimizer, num_epochs, filename):
model.cuda()
for epoch in range(num_epochs):
for phase in ['train', 'val']:
if phase == 'train':
model.train()
else:
model.eval()
epoch_loss = 0.0
epoch_jaccard = 0.0
for data in(dataloaders_dict[phase]):
ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_idx = data['start_idx'].cuda()
end_idx = data['end_idx'].cuda()
optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
start_logits, end_logits = model(ids, masks)
 loss = criterion(start_logits, end_logits, start_idx, end_idx)
 if phase == 'train':
loss.backward()
optimizer.step()
epoch_loss += loss.item() * len(ids)
 start_idx = start_idx.cpu().detach().numpy()
end_idx = end_idx.cpu().detach().numpy()
start_logits = torch.softmax(start_logits, dim=1 ).cpu().detach().numpy()
end_logits = torch.softmax(end_logits, dim=1 ).cpu().detach().numpy()
for i in range(len(ids)) :
jaccard_score = compute_jaccard_score(
 tweet[i],
start_idx[i],
end_idx[i],
start_logits[i],
end_logits[i],
offsets[i])
 epoch_jaccard += jaccard_score
epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)
 epoch_jaccard = epoch_jaccard / len(dataloaders_dict[phase].dataset)
 print('Epoch {}/{} | {:^5} | Loss: {:.4f} | Jaccard: {:.4f}'.format(
 epoch + 1, num_epochs, phase, epoch_loss, epoch_jaccard))
torch.save(model.state_dict() , filename )<choose_model_class>
print('Negative and Tumor Probabilities: ' + str(preds_n_avg[0]))
tumor_preds = preds_n_avg[:, 1]
print('Tumor probability: ' + str(tumor_preds[0]))
class_preds = np.argmax(preds_n_avg, axis=1)
 classes = ['Negative','Tumor']
print('Class prediction: ' + classes[class_preds[0]] )
<load_from_csv><EOS> 
SAMPLE_SUB = '/kaggle/input/sample_submission.csv'
sample_df = pd.read_csv(SAMPLE_SUB)
 sample_list = list(sample_df.id)
 pred_list = [p for p in tumor_preds]
pred_dic = dict(( key, value)for(key, value)in zip(learner.data.test_ds.items, pred_list))
pred_list_cor = [pred_dic['///kaggle/input/test/' + id + '.tif'] for id in sample_list]
df_sub = pd.DataFrame({'id':sample_list,'label':pred_list_cor})
 df_sub.to_csv('{0}_submission.csv'.format(MODEL_PATH), header=True, index=False )