Kaggle competition: SIIM-ACR Pneumothorax Segmentation
<SOS> metric: SIIMDice Kaggle data source: siim-acr-pneumothorax-segmentation<predict_on_test>
!pip install -U git+https://github.com/albu/albumentations
titanic_test_data_X['Survived'] = voting_est.predict(Stacked_test )<create_dataframe>
warnings.filterwarnings("ignore" )
Stacked_train_df = pd.DataFrame(Stacked_train)
 Stacked_test_df = pd.DataFrame(Stacked_test)
 titanic_comb_X = pd.concat([Stacked_train_df, Stacked_test_df])
 titanic_comb_y = pd.concat([titanic_train_data_y, titanic_test_data_X['Survived']] )<choose_model_class>
import albumentations as A
rf_est = ensemble.RandomForestClassifier(n_estimators = 750, criterion = 'gini', max_features = 'sqrt', max_depth = 3, min_samples_split = 4, min_samples_leaf = 2, n_jobs = 35, random_state = 42)
 gbm_est = ensemble.GradientBoostingClassifier(n_estimators = 900, learning_rate = 0.001, loss = 'exponential', min_samples_split = 3, min_samples_leaf = 2, max_features ='sqrt', max_depth = 3, random_state = 42)
 et_est = ensemble.ExtraTreesClassifier(n_estimators = 750, max_depth = 35, n_jobs = 15, criterion = 'entropy', random_state = 42)
 voting_est = ensemble.VotingClassifier(estimators = [('rf', rf_est),('gbm', gbm_est),('et', et_est)],
voting = 'hard',
n_jobs = 35)
 voting_est.fit(titanic_comb_X, titanic_comb_y)
 print("VotingClassifier Score: " + str(voting_est.score(titanic_comb_X, titanic_comb_y)))
 print("VotingClassifier Estimators: " + str(voting_est.estimators_))
titanic_test_data_X['Survived_new'] = voting_est.predict(Stacked_test_df )<save_to_csv>
A.MultiplicativeNoise()
submission = pd.DataFrame({'PassengerId': test_data_orig.loc[:, 'PassengerId'],
'Survived': titanic_test_data_X.loc[:, 'Survived_new']})
 submission.to_csv(".. /working/submission.csv", index = False )<import_modules>
!pip install git+https://github.com/qubvel/segmentation_models.pytorch > /dev/null 2>&1
import pandas as pd
import numpy as np
from sklearn.linear_model import LogisticRegression
import matplotlib.pyplot as plt
from sklearn.utils import shuffle
from sklearn.model_selection import GridSearchCV<load_from_csv>
import segmentation_models_pytorch as smp
tourney_result = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneyCompactResults.csv')
 tourney_seed = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneySeeds.csv' )<drop_column>
def run_length_decode(rle, height=1024, width=1024, fill_value=1):
component = np.zeros(( height, width), np.float32)
 component = component.reshape(-1)
 rle = np.array([int(s)for s in rle.strip().split(' ')])
 rle = rle.reshape(-1, 2)
 start = 0
for index, length in rle:
start = start+index
end = start+length
component[start: end] = fill_value
start = end
component = component.reshape(width, height ).T
return component
def run_length_encode(component):
component = component.T.flatten()
start = np.where(component[1:] > component[:-1])[0]+1
end = np.where(component[:-1] > component[1:])[0]+1
length = end-start
rle = []
for i in range(len(length)) :
if i == 0:
rle.extend([start[0], length[0]])
 else:
rle.extend([start[i]-end[i-1], length[i]])
 rle = ' '.join([str(r)for r in rle])
 return rle
tourney_result = tourney_result.drop(['DayNum', 'WScore', 'LScore', 'WLoc', 'NumOT'], axis=1)
 tourney_result<merge>
class SIIMDataset(Dataset):
def __init__(self, df, fnames, data_folder, size, mean, std, phase):
self.df = df
self.root = data_folder
self.size = size
self.mean = mean
self.std = std
self.phase = phase
self.transforms = get_transforms(phase, size, mean, std)
 self.gb = self.df.groupby('ImageId')
 self.fnames = fnames
def __getitem__(self, idx):
image_id = self.fnames[idx]
df = self.gb.get_group(image_id)
 annotations = df[' EncodedPixels'].tolist()
image_path = os.path.join(self.root, image_id + ".png")
 image = cv2.imread(image_path)
 mask = np.zeros([1024, 1024])
 if annotations[0] != '-1':
for rle in annotations:
mask += run_length_decode(rle)
 mask =(mask >= 1 ).astype('float32')
 augmented = self.transforms(image=image, mask=mask)
 image = augmented['image']
mask = augmented['mask']
return image, mask
def __len__(self):
return len(self.fnames)
 def get_transforms(phase, size, mean, std):
list_transforms = []
if phase == "train":
list_transforms.extend(
 [
HorizontalFlip(p=0.5),
ShiftScaleRotate(
 shift_limit=0,
scale_limit=0.1,
rotate_limit=10,
p=0.5,
border_mode=cv2.BORDER_CONSTANT 
),
GaussNoise() ,
A.MultiplicativeNoise(multiplier=1.5, p=1),
] 
)
 list_transforms.extend(
 [
Resize(size, size),
Normalize(mean=mean, std=std, p=1),
ToTensor() ,
] 
)
 list_trfms = Compose(list_transforms)
 return list_trfms
def provider(
 fold,
total_folds,
data_folder,
df_path,
phase,
size,
mean=None,
std=None,
batch_size=8,
num_workers=2, 
):
df_all = pd.read_csv(df_path)
 df = df_all.drop_duplicates('ImageId')
 df_with_mask = df[df[" EncodedPixels"] != "-1"]
df_with_mask['has_mask'] = 1
df_without_mask = df[df[" EncodedPixels"] == "-1"]
df_without_mask['has_mask'] = 0
df_without_mask_sampled = df_without_mask.sample(len(df_with_mask)+1500, random_state=2019)
 df = pd.concat([df_with_mask, df_without_mask_sampled])
 kfold = StratifiedKFold(total_folds, shuffle=True, random_state=43)
 train_idx, val_idx = list(kfold.split(df["ImageId"], df["has_mask"])) [fold]
train_df, val_df = df.iloc[train_idx], df.iloc[val_idx]
df = train_df if phase == "train" else val_df
fnames = df['ImageId'].values
image_dataset = SIIMDataset(df_all, fnames, data_folder, size, mean, std, phase)
 dataloader = DataLoader(
 image_dataset,
batch_size=batch_size,
num_workers=num_workers,
pin_memory=True,
shuffle=True, 
)
 return dataloader
tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Seed':'WSeed'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Seed':'LSeed'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result<categorify>
sample_submission_path = '.. /input/siimmy/stage_2_sample_submission.csv'
train_rle_path = '.. /input/mysiim/train-rle.csv'
data_folder = ".. /input/siimpng/siimpng/train_png"
test_data_folder = ".. /input/siim_stage2_png"
def get_seed(x):
return int(x[1:3])
 tourney_result['WSeed'] = tourney_result['WSeed'].map(lambda x: get_seed(x))
tourney_result['LSeed'] = tourney_result['LSeed'].map(lambda x: get_seed(x))
tourney_result<load_from_csv>
ab = glob.glob('.. /input/siimpng/siimpng/train_png/*.png')
 len(ab )
season_result = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MRegularSeasonCompactResults.csv' )<concatenate>
a= pd.read_csv(train_rle_path)
 len(a )
season_win_result = season_result[['Season', 'WTeamID', 'WScore']]
season_lose_result = season_result[['Season', 'LTeamID', 'LScore']]
season_win_result.rename(columns={'WTeamID':'TeamID', 'WScore':'Score'}, inplace=True)
 season_lose_result.rename(columns={'LTeamID':'TeamID', 'LScore':'Score'}, inplace=True)
 season_result = pd.concat(( season_win_result, season_lose_result)).reset_index(drop=True)
 season_result<groupby>
dataloader = provider(
 fold=0,
total_folds=5,
data_folder=data_folder,
df_path=train_rle_path,
phase="train",
size=512,
mean =(0.485, 0.456, 0.406),
std =(0.229, 0.224, 0.225),
batch_size=16,
num_workers=2,
)
season_score = season_result.groupby(['Season', 'TeamID'])['Score'].sum().reset_index()
season_score<merge>
batch = next(iter(dataloader))
images, masks = batch
tourney_result = pd.merge(tourney_result, season_score, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Score':'WScoreT'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result = pd.merge(tourney_result, season_score, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Score':'LScoreT'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result<drop_column>
def dice_loss(input, target):
input = torch.sigmoid(input)
 smooth = 1.0
iflat = input.view(-1)
 tflat = target.view(-1)
 intersection =(iflat * tflat ).sum()
return(( 2.0 * intersection + smooth)/(iflat.sum() + tflat.sum() + smooth))
class FocalLoss(nn.Module):
def __init__(self, gamma):
super().__init__()
self.gamma = gamma
def forward(self, input, target):
if not(target.size() == input.size()):
raise ValueError("Target size({})must be the same as input size({})" 
.format(target.size() , input.size()))
max_val =(-input ).clamp(min=0)
 loss = input - input * target + max_val + \ 
(( -max_val ).exp() +(-input - max_val ).exp() ).log()
invprobs = F.logsigmoid(-input *(target * 2.0 - 1.0))
loss =(invprobs * self.gamma ).exp() * loss
return loss.mean()
class MixedLoss(nn.Module):
def __init__(self, alpha, gamma):
super().__init__()
self.alpha = alpha
self.focal = FocalLoss(gamma)
 def forward(self, input, target):
loss = self.alpha*self.focal(input, target)- torch.log(dice_loss(input, target))
return loss.mean()
tourney_win_result = tourney_result.drop(['Season', 'WTeamID', 'LTeamID'], axis=1)
 tourney_win_result.rename(columns={'WSeed':'Seed1', 'LSeed':'Seed2', 'WScoreT':'ScoreT1', 'LScoreT':'ScoreT2'}, inplace=True)
 tourney_win_result<prepare_output>
def predict(X, threshold):
X_p = np.copy(X)
 preds =(X_p > threshold ).astype('uint8')
 return preds
def metric(probability, truth, threshold=0.5, reduction='none'):


batch_size = len(truth)
 with torch.no_grad() :
probability = probability.view(batch_size, -1)
 truth = truth.view(batch_size, -1)
 assert(probability.shape == truth.shape)
 p =(probability > threshold ).float()
t =(truth > 0.5 ).float()
t_sum = t.sum(-1)
 p_sum = p.sum(-1)
 neg_index = torch.nonzero(t_sum == 0)
 pos_index = torch.nonzero(t_sum >= 1)
 dice_neg =(p_sum == 0 ).float()
dice_pos = 2 *(p*t ).sum(-1)/(( p+t ).sum(-1))
dice_neg = dice_neg[neg_index]
dice_pos = dice_pos[pos_index]
dice = torch.cat([dice_pos, dice_neg])
 dice_neg = np.nan_to_num(dice_neg.mean().item() , 0)
 dice_pos = np.nan_to_num(dice_pos.mean().item() , 0)
 dice = dice.mean().item()
num_neg = len(neg_index)
 num_pos = len(pos_index)
 return dice, dice_neg, dice_pos, num_neg, num_pos
class Meter:

 def __init__(self, phase, epoch):
self.base_threshold = 0.5
self.base_dice_scores = []
self.dice_neg_scores = []
self.dice_pos_scores = []
self.iou_scores = []
def update(self, targets, outputs):
probs = torch.sigmoid(outputs)
 dice, dice_neg, dice_pos, _, _ = metric(probs, targets, self.base_threshold)
 self.base_dice_scores.append(dice)
 self.dice_pos_scores.append(dice_pos)
 self.dice_neg_scores.append(dice_neg)
 preds = predict(probs, self.base_threshold)
 iou = compute_iou_batch(preds, targets, classes=[1])
 self.iou_scores.append(iou)
 def get_metrics(self):
dice = np.mean(self.base_dice_scores)
 dice_neg = np.mean(self.dice_neg_scores)
 dice_pos = np.mean(self.dice_pos_scores)
 dices = [dice, dice_neg, dice_pos]
iou = np.nanmean(self.iou_scores)
 return dices, iou
def epoch_log(phase, epoch, epoch_loss, meter, start):

 dices, iou = meter.get_metrics()
dice, dice_neg, dice_pos = dices
print("Loss: %0.4f | dice: %0.4f | dice_neg: %0.4f | dice_pos: %0.4f | IoU: %0.4f" %(epoch_loss, dice, dice_neg, dice_pos, iou))
return dice, iou
def compute_ious(pred, label, classes, ignore_index=255, only_present=True):

 pred[label == ignore_index] = 0
ious = []
for c in classes:
label_c = label == c
if only_present and np.sum(label_c)== 0:
ious.append(np.nan)
 continue
pred_c = pred == c
intersection = np.logical_and(pred_c, label_c ).sum()
union = np.logical_or(pred_c, label_c ).sum()
if union != 0:
ious.append(intersection / union)
 return ious if ious else [1]
def compute_iou_batch(outputs, labels, classes=None):

 ious = []
preds = np.copy(outputs)
 labels = np.array(labels)
 for pred, label in zip(preds, labels):
ious.append(np.nanmean(compute_ious(pred, label, classes)))
 iou = np.nanmean(ious)
 return iou
tourney_lose_result = tourney_win_result.copy()
tourney_lose_result['Seed1'] = tourney_win_result['Seed2']
tourney_lose_result['Seed2'] = tourney_win_result['Seed1']
tourney_lose_result['ScoreT1'] = tourney_win_result['ScoreT2']
tourney_lose_result['ScoreT2'] = tourney_win_result['ScoreT1']
tourney_lose_result<feature_engineering>
model = smp.FPN('inceptionresnetv2', encoder_weights="imagenet" )
tourney_win_result['Seed_diff'] = tourney_win_result['Seed1'] - tourney_win_result['Seed2']
tourney_win_result['ScoreT_diff'] = tourney_win_result['ScoreT1'] - tourney_win_result['ScoreT2']
tourney_lose_result['Seed_diff'] = tourney_lose_result['Seed1'] - tourney_lose_result['Seed2']
tourney_lose_result['ScoreT_diff'] = tourney_lose_result['ScoreT1'] - tourney_lose_result['ScoreT2']<concatenate>
class RAdam(Optimizer):
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0):
defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay)
 self.buffer = [[None, None, None] for ind in range(10)]
super(RAdam, self ).__init__(params, defaults)
 def __setstate__(self, state):
super(RAdam, self ).__setstate__(state)
 def step(self, closure=None):
loss = None
if closure is not None:
loss = closure()
for group in self.param_groups:
for p in group['params']:
if p.grad is None:
continue
grad = p.grad.data.float()
if grad.is_sparse:
raise RuntimeError('RAdam does not support sparse gradients')
 p_data_fp32 = p.data.float()
state = self.state[p]
if len(state)== 0:
state['step'] = 0
state['exp_avg'] = torch.zeros_like(p_data_fp32)
 state['exp_avg_sq'] = torch.zeros_like(p_data_fp32)
 else:
state['exp_avg'] = state['exp_avg'].type_as(p_data_fp32)
 state['exp_avg_sq'] = state['exp_avg_sq'].type_as(p_data_fp32)
 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
beta1, beta2 = group['betas']
exp_avg_sq.mul_(beta2 ).addcmul_(1 - beta2, grad, grad)
 exp_avg.mul_(beta1 ).add_(1 - beta1, grad)
 state['step'] += 1
buffered = self.buffer[int(state['step'] % 10)]
if state['step'] == buffered[0]:
N_sma, step_size = buffered[1], buffered[2]
else:
buffered[0] = state['step']
beta2_t = beta2 ** state['step']
N_sma_max = 2 /(1 - beta2)- 1
N_sma = N_sma_max - 2 * state['step'] * beta2_t /(1 - beta2_t)
 buffered[1] = N_sma
if N_sma >= 5:
step_size = group['lr'] * math.sqrt(( 1 - beta2_t)*(N_sma - 4)/(N_sma_max - 4)*(N_sma - 2)/ N_sma * N_sma_max /(N_sma_max - 2)) /(1 - beta1 ** state['step'])
 else:
step_size = group['lr'] /(1 - beta1 ** state['step'])
 buffered[2] = step_size
if group['weight_decay'] != 0:
p_data_fp32.add_(-group['weight_decay'] * group['lr'], p_data_fp32)
 if N_sma >= 5:
denom = exp_avg_sq.sqrt().add_(group['eps'])
 p_data_fp32.addcdiv_(-step_size, exp_avg, denom)
 else:
p_data_fp32.add_(-step_size, exp_avg)
 p.data.copy_(p_data_fp32)
 return loss
tourney_win_result['result'] = 1
tourney_lose_result['result'] = 0
tourney_result = pd.concat(( tourney_win_result, tourney_lose_result)).reset_index(drop=True)
 tourney_result<load_from_csv>
class Trainer(object):

 def __init__(self, model):
self.fold = 1
self.total_folds = 5
self.num_workers = 4
self.batch_size = {"train": 4, "val": 4}
self.accumulation_steps = 32 // self.batch_size['train']
self.lr = 5e-4
self.num_epochs = 32
self.best_loss = float("inf")
 self.phases = ["train", "val"]
self.device = torch.device("cuda:0")
 torch.set_default_tensor_type("torch.cuda.FloatTensor")
 self.net = model
self.criterion = MixedLoss(10.0, 2.0)
 self.optimizer = RAdam(model.parameters() , lr=self.lr)
 self.scheduler = ReduceLROnPlateau(self.optimizer, mode="min", patience=3, verbose=True)
 self.net = self.net.to(self.device)
 cudnn.benchmark = True
self.dataloaders = {
phase: provider(
 fold=1,
total_folds=5,
data_folder=data_folder,
df_path=train_rle_path,
phase=phase,
size=512,
mean=(0.485, 0.456, 0.406),
std=(0.229, 0.224, 0.225),
batch_size=self.batch_size[phase],
num_workers=self.num_workers, 
)
 for phase in self.phases
}
self.losses = {phase: [] for phase in self.phases}
self.iou_scores = {phase: [] for phase in self.phases}
self.dice_scores = {phase: [] for phase in self.phases}
def forward(self, images, targets):
images = images.to(self.device)
 masks = targets.to(self.device)
 outputs = self.net(images)
 loss = self.criterion(outputs, masks)
 return loss, outputs
def iterate(self, epoch, phase):
meter = Meter(phase, epoch)
 start = time.strftime("%H:%M:%S")
 print(f"Starting epoch: {epoch} | phase: {phase} | ⏰: {start}")
 batch_size = self.batch_size[phase]
self.net.train(phase == "train")
 dataloader = self.dataloaders[phase]
running_loss = 0.0
total_batches = len(dataloader)
 self.optimizer.zero_grad()
for itr, batch in enumerate(dataloader):
images, targets = batch
loss, outputs = self.forward(images, targets)
 loss = loss / self.accumulation_steps
if phase == "train":
loss.backward()
if(itr + 1)% self.accumulation_steps == 0:
self.optimizer.step()
self.optimizer.zero_grad()
running_loss += loss.item()
outputs = outputs.detach().cpu()
meter.update(targets, outputs)
 epoch_loss =(running_loss * self.accumulation_steps)/ total_batches
dice, iou = epoch_log(phase, epoch, epoch_loss, meter, start)
 self.losses[phase].append(epoch_loss)
 self.dice_scores[phase].append(dice)
 self.iou_scores[phase].append(iou)
 torch.cuda.empty_cache()
return epoch_loss
def start(self):
for epoch in range(self.num_epochs):
self.iterate(epoch, "train")
 state = {
"epoch": epoch,
"best_loss": self.best_loss,
"state_dict": self.net.state_dict() ,
"optimizer": self.optimizer.state_dict() ,
}
val_loss = self.iterate(epoch, "val")
 self.scheduler.step(val_loss)
 if val_loss < self.best_loss:
print("******** New optimal found, saving state ********")
 state["best_loss"] = self.best_loss = val_loss
torch.save(state, "./model.pth")
 print()
test_df = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MSampleSubmissionStage1_2020.csv' )<feature_engineering>
model_trainer = Trainer(model)
 model_trainer.start()
test_df['Season'] = test_df['ID'].map(lambda x: int(x[:4]))
test_df['WTeamID'] = test_df['ID'].map(lambda x: int(x[5:9]))
test_df['LTeamID'] = test_df['ID'].map(lambda x: int(x[10:14]))
test_df<merge>
class TestDataset(Dataset):
def __init__(self, root, df, size, mean, std, tta=4):
self.root = root
self.size = size
self.fnames = list(df["ImageId"])
 self.num_samples = len(self.fnames)
 self.transform = Compose(
 [
Normalize(mean=mean, std=std, p=1),
Resize(size, size),
ToTensor() ,
] 
)
 def __getitem__(self, idx):
fname = self.fnames[idx]
path = os.path.join(self.root, fname + ".png")
 image = cv2.imread(path)
 images = self.transform(image=image)["image"]
return images
def __len__(self):
return self.num_samples
def post_process(probability, threshold, min_size):
mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]
num_component, component = cv2.connectedComponents(mask.astype(np.uint8))
predictions = np.zeros(( 1024, 1024), np.float32)
 num = 0
for c in range(1, num_component):
p =(component == c)
 if p.sum() > min_size:
predictions[p] = 1
num += 1
return predictions, num
<feature_engineering><EOS> 
size = 512
mean =(0.485, 0.456, 0.406)
 std =(0.229, 0.224, 0.225)
 num_workers = 8
batch_size = 16
best_threshold = 0.5
min_size = 3500
device = torch.device("cuda:0")
 df = pd.read_csv(sample_submission_path)
 testset = DataLoader(
 TestDataset(test_data_folder, df, size, mean, std),
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=True, 
)
 model = model_trainer.net
model.eval()
state = torch.load('./model.pth', map_location=lambda storage, loc: storage)
 model.load_state_dict(state["state_dict"])
 encoded_pixels = []
for i, batch in enumerate(tqdm(testset)) :
preds = torch.sigmoid(model(batch.to(device)))
 preds = preds.detach().cpu().numpy() [:, 0, :, :]
for probability in preds:
if probability.shape !=(1024, 1024):
probability = cv2.resize(probability, dsize=(1024, 1024), interpolation=cv2.INTER_LINEAR)
 predict, num_predict = post_process(probability, best_threshold, min_size)
 if num_predict == 0:
encoded_pixels.append('-1')
 else:
r = run_length_encode(predict)
 encoded_pixels.append(r)
 df['EncodedPixels'] = encoded_pixels
df.to_csv('submission.csv', columns=['ImageId', 'EncodedPixels'], index=False )