Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<data_type_conversions>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from fastai.vision import *
from fastai.metrics import *
from fastai import *
from os import *
import matplotlib.pyplot as plt
import matplotlib.patches as patches
import random
from sklearn.utils import shuffle
class TweetDataset(Dataset):
def __init__(self, df, tokenizer):
self.text = df['text'].tolist()
self.selected_text = df['selected_text'].tolist()
self.target_start_idx = df['target_start_idx'].tolist()
self.target_end_idx = df['target_end_idx'].tolist()
self.sentiment = df['sentiment'].tolist()
self.tokenizer = tokenizer
def __len__(self):
return len(self.text)
 def __getitem__(self, idx):
encoded = self._process_data(self.text[idx], self.sentiment[idx], self.target_start_idx[idx], self.target_end_idx[idx])
 data = {
'input_ids': torch.tensor(encoded['input_ids'], dtype=torch.long),
'token_type_ids': torch.tensor(encoded['token_type_ids'], dtype=torch.long),
'attention_mask': torch.tensor(encoded['attention_mask'], dtype=torch.float),
'target_start_idx': torch.tensor(encoded['target_start_idx'], dtype=torch.long),
'target_end_idx': torch.tensor(encoded['target_end_idx'], dtype=torch.long)
 }
return data
def _process_data(self, text, sentiment, target_start_idx, target_end_idx):
data = self.tokenizer.encode_plus(sentiment, text, pad_to_max_length=True)
 data['target_start_idx'] = target_start_idx + 3
data['target_end_idx'] = target_end_idx + 3
return data<normalization>
model_path='.'
path='.. /input/'
train_folder=f'{path}train'
test_folder=f'{path}test'
train_lbl=f'{path}train_labels.csv'
ORG_SIZE=96
bs=64
num_workers=None
sz=96
class SentimentExtractionBert(nn.Module):
def __init__(self):
super(SentimentExtractionBert, self ).__init__()
self.num_labels = 2
self.bert = BertModel.from_pretrained('/kaggle/input/bertbaseuncased/')
 self.dropout = nn.Dropout(0.2)
 self.outputs = nn.Linear(768 * 2, self.num_labels)
 nn.init.zeros_(self.outputs.bias)
 nn.init.normal_(self.outputs.weight, mean=0.0, std=0.02)
 def forward(self,
input_ids,
attention_mask,
token_type_ids, 
):
outputs = self.bert(
 input_ids=input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids 
)
 sequence_output = outputs[0]
sentiment_rep = sequence_output[:,1].unsqueeze(1 ).repeat(1, 192, 1)
 sent_seq_output = torch.cat([sequence_output, sentiment_rep], dim=-1)
 logits = self.outputs(self.dropout(sent_seq_output))
start_logits, end_logits = logits.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<define_variables>
df_train=pd.read_csv(train_lbl )
FOLD = 5
EPOCHS = 3
BATCH_SIZE = 32
LEARNING_RATE = 3e-5
WEIGHT_DECAY = 0.01
WARMUP_STEPS = 0
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu' )<split>
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=.0, max_zoom=1.1,
max_lighting=0.05, max_warp=0.)
kf = KFold(n_splits=FOLD)
 kfold_list = list(kf.split(train_dataset))<train_on_grid>
data = ImageDataBunch.from_csv(path,csv_labels=train_lbl,folder='train',valid_pct=0.2, ds_tfms=tfms, size=sz, suffix='.tif',test=test_folder,bs=bs);
stats=data.batch_stats()
data.normalize(stats )
def train_one_epoch(model, train_dl, optimizer, scheduler):
criterion = nn.CrossEntropyLoss()
training_loss = 0
model.train()
for data in tqdm(train_dl):
input_ids = data['input_ids'].to(device)
 token_type_ids = data['token_type_ids'].to(device)
 attention_mask = data['attention_mask'].to(device)
 target_start_idx = data['target_start_idx'].to(device)
 target_end_idx = data['target_end_idx'].to(device)
 optimizer.zero_grad()
start_logits, end_logits = model(input_ids, attention_mask, token_type_ids)
 loss = criterion(start_logits, target_start_idx)+ criterion(end_logits, target_end_idx)
 loss.backward()
optimizer.step()
scheduler.step()
training_loss += loss.item()
training_loss /= len(train_dl)
 return training_loss<train_on_grid>
data.show_batch(rows=5, figsize=(12,9))
def train(fold):
model = SentimentExtractionBert()
model = model.to(device)
 train_ds = TweetDataset(train_dataset.iloc[kfold_list[fold][0]], tokenizer)
 train_dl = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, drop_last=True)
 no_decay = ["bias", "LayerNorm.weight"]
optimizer_grouped_parameters = [
{
"params": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
"weight_decay": WEIGHT_DECAY,
},
{ "params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],
"weight_decay": 0.0
},
]
optimizer = AdamW(optimizer_grouped_parameters, lr=LEARNING_RATE, betas=(0.9, 0.98), eps=1e-8)
 t_total = len(train_dl)* EPOCHS
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=WARMUP_STEPS, num_training_steps=t_total)
 for epoch in range(EPOCHS):
training_loss = train_one_epoch(model, train_dl, optimizer, scheduler)
 with open("./log.txt", 'a')as f:
f.write("Fold {}, Epoch {}, Learning Rate: {},Loss: {}
".format(fold, epoch, optimizer.param_groups[0]['lr'], training_loss))
torch.save(model.state_dict() , 'ft_ckpt_{}.pth'.format(fold))<train_model>
from sklearn.metrics import roc_auc_score
for fold in range(FOLD):
train(fold )<find_best_params>
def auc_score(y_pred,y_true,tens=True):
score=roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])
 if tens:
score=tensor(score)
 else:
score=score
return score
model0 = SentimentExtractionBert()
model0.load_state_dict(torch.load("ft_ckpt_0.pth"))
model0.to(device)
 model0.eval()
model1 = SentimentExtractionBert()
model1.load_state_dict(torch.load("ft_ckpt_1.pth"))
model1.to(device)
 model1.eval()
model2 = SentimentExtractionBert()
model2.load_state_dict(torch.load("ft_ckpt_2.pth"))
model2.to(device)
 model2.eval()
model3 = SentimentExtractionBert()
model3.load_state_dict(torch.load("ft_ckpt_3.pth"))
model3.to(device)
 model3.eval()
model4 = SentimentExtractionBert()
model4.load_state_dict(torch.load("ft_ckpt_4.pth"))
model4.to(device)
 model4.eval() 
<categorify>
learn = create_cnn(data, models.resnet34, metrics=[auc_score], model_dir="/tmp/model/", ps=0.5 )
def process_test_data(row):
data = tokenizer.encode_plus(row['sentiment'], row['text'], pad_to_max_length=True)
 tensor_data = {
'input_ids': torch.tensor(data['input_ids'], dtype=torch.long ).unsqueeze(0),
'token_type_ids': torch.tensor(data['token_type_ids'], dtype=torch.long ).unsqueeze(0),
'attention_mask': torch.tensor(data['attention_mask'], dtype=torch.float ).unsqueeze(( 0)) ,
}
return tensor_data
def postprocessing(text):
out_text =(
 text.replace(" ` ", "`")
.replace("* * * *", "****")
)
 return out_text
def test(row):
if row['sentiment'] == 'neutral':
predicted_text = row['text']
else:
data = process_test_data(row)
 input_ids = data['input_ids'].to(device)
 token_type_ids = data['token_type_ids'].to(device)
 attention_mask = data['attention_mask'].to(device)
 with torch.no_grad() :
start_logits_0, end_logits_0 = model0(input_ids, attention_mask, token_type_ids)
 start_logits_1, end_logits_1 = model1(input_ids, attention_mask, token_type_ids)
 start_logits_2, end_logits_2 = model2(input_ids, attention_mask, token_type_ids)
 start_logits_3, end_logits_3 = model3(input_ids, attention_mask, token_type_ids)
 start_logits_4, end_logits_4 = model4(input_ids, attention_mask, token_type_ids)
 start_logits =(
 start_logits_0
+ start_logits_1
+ start_logits_2
+ start_logits_3
+ start_logits_4 
)/ 5
end_logits =(
 end_logits_0
+ end_logits_1
+ end_logits_2
+ end_logits_3
+ end_logits_4 
)/ 5
predicted_start_idx = torch.argmax(torch.softmax(start_logits, dim=1), dim=1 ).item()
predicted_end_idx = torch.argmax(torch.softmax(end_logits, dim=1), dim=1 ).item()
input_ids = input_ids.squeeze()
predicted_text = tokenizer.decode(input_ids[predicted_start_idx : predicted_end_idx + 1], skip_special_tokens=True)
 predicted_text = postprocessing(predicted_text)
 return predicted_text<load_from_csv>
learn.fit_one_cycle(1, 1e-3 )
test_dataset = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/test.csv')
 test_dataset['predicted_text'] = test_dataset.progress_apply(test, axis=1 )<save_to_csv>
learn.lr_find()
submission = pd.read_csv('/kaggle/input/tweet-sentiment-extraction/sample_submission.csv')
 submission['selected_text'] = test_dataset['predicted_text']
submission.to_csv('./submission.csv', index=False )<set_options>
learn.fit_one_cycle(5, slice(1e-4,1e-1))
with warnings.catch_warnings() :
warnings.filterwarnings("ignore")
<define_variables>
learn.save('stage-1' )
RUN_LOCAL = os.environ.get("PWD")!= "/kaggle/working"
RUN_LOCAL<define_variables>
interp = ClassificationInterpretation.from_learner(learn)
 losses,idxs = interp.top_losses()
len(data.valid_ds)==len(losses)==len(idxs )
SEED = 42
if RUN_LOCAL:
NUM_EPOCHS = 1
HOME_DIR = "/workspaces/kaggle"
BATCH_SIZE = 1
SKF = StratifiedKFold(n_splits=2, shuffle=True, random_state=SEED)
 else:
NUM_EPOCHS = 3
HOME_DIR = "/kaggle"
BATCH_SIZE = 32
SKF = StratifiedKFold(n_splits=10, shuffle=True, random_state=SEED )<set_options>
learn.fit_one_cycle(1 )
def seed_everything(seed_value: int)-> None:

 random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ["PYTHONHASHSEED"] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )<categorify>
learn.fit_one_cycle(5, slice(5e-5,5e-3))
def _get_input_data(
 row: pd.Series,
tokenizer: tokenizers.ByteLevelBPETokenizer,
max_len: int,
bos_token_id: int,
eos_token_id: int, 
)-> Tuple[torch.Tensor, torch.Tensor, str, torch.Tensor]:

 tweet = " " + " ".join(row.text.lower().split())
 encoding = tokenizer.encode(tweet)
 sentiment_id = tokenizer.encode(row.sentiment ).ids
ids =(
 [bos_token_id]
+ sentiment_id
+ [eos_token_id, eos_token_id]
+ encoding.ids
+ [eos_token_id] 
)
 offsets = [(0, 0)] * 4 + encoding.offsets + [(0, 0)]
pad_len = max_len - len(ids)
 if pad_len > 0:
ids += [1] * pad_len
offsets += [(0, 0)] * pad_len
ids = torch.tensor(ids)
 masks = torch.where(ids != 1, torch.tensor(1), torch.tensor(0))
offsets = torch.tensor(offsets)
 assert ids.shape == torch.Size([96])
 assert masks.shape == torch.Size([96])
 assert offsets.shape == torch.Size([96, 2])
 return ids, masks, tweet, offsets<categorify>
learn.save('stage-2' )
def _get_target_idx(row: pd.Series, tweet: str, offsets: torch.Tensor)-> Tuple[int, int]:

 assert offsets.shape == torch.Size([96, 2])
 selected_text = " " + " ".join(row.selected_text.lower().split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind : ind + len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st
break
char_targets = [0] * len(tweet)
 if idx0 is not None and idx1 is not None:
for ct in range(idx0, idx1):
char_targets[ct] = 1
target_idx: List[int] = []
for j,(offset1, offset2)in enumerate(offsets):
left = offset1.item()
right = offset2.item()
assert isinstance(left, int)
 assert isinstance(right, int)
 if sum(char_targets[left:right])> 0:
target_idx.append(j)
 start_idx = target_idx[0]
end_idx = target_idx[-1]
return start_idx, end_idx<categorify>
interp = ClassificationInterpretation.from_learner(learn)
 losses,idxs = interp.top_losses()
len(data.valid_ds)==len(losses)==len(idxs )
def _process_row(
 row: pd.Series,
tokenizer: tokenizers.ByteLevelBPETokenizer,
max_len: int,
bos_token_id: int,
eos_token_id: int, 
)-> Dict[str, Union[torch.Tensor, str, int]]:

 data: Dict[str, Union[torch.Tensor, str, int]] = {}
ids, masks, tweet, offsets = _get_input_data(
 row, tokenizer, max_len, bos_token_id, eos_token_id 
)
 data.update({"ids": ids})
 data["masks"] = masks
data["tweet"] = tweet
data["offsets"] = offsets
if "selected_text" in row:
start_idx, end_idx = _get_target_idx(row, tweet, offsets)
 data["start_idx"] = start_idx
data["end_idx"] = end_idx
return data<create_dataframe>
preds,y=learn.get_preds()
pred_score=auc_score(preds,y)
 pred_score
class TweetDataset(Dataset):

 def __init__(self, df: pd.DataFrame, max_len: int = 96)-> None:

 self.df = df
self.max_len = max_len
self.labeled = "selected_text" in df
tokenizer = RobertaTokenizer.from_pretrained(f"{HOME_DIR}/input/roberta-base")
 tokenizer.save_vocabulary(".")
 self.bos_token_id = tokenizer.bos_token_id
self.eos_token_id = tokenizer.eos_token_id
self.tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file="./vocab.json",
merges_file="./merges.txt",
lowercase=True,
add_prefix_space=True, 
)
 def __getitem__(self, index: int)-> Dict[str, Union[torch.Tensor, str, int]]:

 row = self.df.iloc[index]
data = _process_row(
 row, self.tokenizer, self.max_len, self.bos_token_id, self.eos_token_id 
)
 return data
def __len__(self)-> int:

 return len(self.df)
 def get_train_val_loaders(
 df: pd.DataFrame, train_idx: np.array, val_idx: np.array, batch_size: int = 8 
)-> Dict[str, torch.utils.data.DataLoader]:

 train_df = df.iloc[train_idx]
val_df = df.iloc[val_idx]
train_loader = torch.utils.data.DataLoader(
 TweetDataset(train_df),
batch_size=batch_size,
shuffle=True,
num_workers=2,
drop_last=True, 
)
 val_loader = torch.utils.data.DataLoader(
 TweetDataset(val_df), batch_size=batch_size, shuffle=False, num_workers=2, 
)
 dataloaders_dict = {"train": train_loader, "val": val_loader}
return dataloaders_dict
def get_test_loader(df: pd.DataFrame, batch_size: int = 32)-> torch.utils.data.DataLoader:

 loader = torch.utils.data.DataLoader(
 TweetDataset(df), batch_size=batch_size, shuffle=False, num_workers=2 
)
 return loader<choose_model_class>
preds_test,y_test=learn.get_preds(ds_type=DatasetType.Test )
class TweetModel(nn.Module):

 def __init__(self)-> None:

 super(TweetModel, self ).__init__()
self.roberta = RobertaModel.from_pretrained(
 ".. /input/roberta-base", output_hidden_states=True 
)
 self.dropout = nn.Dropout(0.5)
 self.fc = nn.Linear(self.roberta.config.hidden_size, 2)
 nn.init.normal_(self.fc.weight, std=0.02)
 nn.init.normal_(self.fc.bias, 0)
 def forward(
 self, input_ids: torch.Tensor, attention_mask: torch.Tensor 
)-> Tuple[torch.Tensor, torch.Tensor]:

 assert input_ids.shape[1:] == torch.Size([96])
 assert attention_mask.shape[1:] == torch.Size([96])
 hs: Tuple[torch.Tensor,...]
_, _, hs = self.roberta(input_ids, attention_mask)
 assert len(hs)== 13
assert all(i.shape[1:] == torch.Size([96, 768])for i in hs)
 x = torch.stack([hs[-1], hs[-2], hs[-3], hs[-4]])
 assert x.shape[0] == 4
assert x.shape[2:] == torch.Size([96, 768])
 x = torch.mean(x, 0)
 assert x.shape[1:] == torch.Size([96, 768])
 x = self.dropout(x)
 x = self.fc(x)
 assert x.shape[1:] == torch.Size([96, 2])
 start_logits, end_logits = x.split(1, dim=-1)
 assert all(i.shape[1:] == torch.Size([96, 1])for i in [start_logits, end_logits])
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 assert all(i.shape[1:] == torch.Size([96])for i in [start_logits, end_logits])
 return start_logits, end_logits<predict_on_test>
clean_fname=np.vectorize(lambda fname: str(fname ).split('/')[-1].split('.')[0])
 fname_cleaned=clean_fname(data.test_ds.items)
 fname_cleaned=fname_cleaned.astype(str )
def loss_fn(
 start_logits: torch.Tensor,
end_logits: torch.Tensor,
start_positions: torch.Tensor,
end_positions: torch.Tensor, 
)-> torch.Tensor:

 assert start_logits.shape[1:] == torch.Size([96])
 assert end_logits.shape[1:] == torch.Size([96])
 assert start_positions.shape[1:] == torch.Size([])
 assert end_positions.shape[1:] == torch.Size([])
 ce_loss = nn.CrossEntropyLoss()
start_loss = ce_loss(start_logits, start_positions)
 assert start_loss.shape == torch.Size([])
 end_loss = ce_loss(end_logits, end_positions)
 assert end_loss.shape == torch.Size([])
 total_loss = start_loss + end_loss
return total_loss<define_variables>
sub.loc[fname_cleaned,'label']=to_np(preds_test[:,1])
 sub.to_csv(f'/kaggle/working/submission_{pred_score}.csv' )