Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<feature_engineering>
! pip install albumentations
! pip install pytorchcv
def five_type_remove_drift(train):
a=500000; b=600000
train.loc[train.index[a:b],'signal'] = train.signal[a:b].values - 3*(train.time.values[a:b] - 50)/10.
 def f(x,low,high,mid):
return -(( -low+high)/625)*(x-mid)**2+high -low
batch = 7; a = 500000*(batch-1); b = 500000*batch
train.loc[train.index[a:b],'signal'] = train.signal.values[a:b] - f(train.time[a:b].values,-1.817,3.186,325)
 batch = 8; a = 500000*(batch-1); b = 500000*batch
train.loc[train.index[a:b],'signal'] = train.signal.values[a:b] - f(train.time[a:b].values,-0.094,4.936,375)
 batch = 9; a = 500000*(batch-1); b = 500000*batch
train.loc[train.index[a:b],'signal'] = train.signal.values[a:b] - f(train.time[a:b].values,1.715,6.689,425)
 batch = 10; a = 500000*(batch-1); b = 500000*batch
train.loc[train.index[a:b],'signal'] = train.signal.values[a:b] - f(train.time[a:b].values,3.361,8.45,475)
 return train<prepare_x_and_y>
%matplotlib inline
train_on_gpu = True
cudnn.benchmark = True
if TRAIN:
df_train = five_type_remove_drift(df_train)
 batch = 1; a = 500000*(batch-1); b = 500000*batch
batch = 2; c = 500000*(batch-1); d = 500000*batch
X_train_1s = np.concatenate([df_train.signal.values[a:b],df_train.signal.values[c:d]] ).reshape(( -1,1))
y_train_1s = np.concatenate([df_train.open_channels.values[a:b],df_train.open_channels.values[c:d]] ).reshape(( -1,1))
batch = 3; a = 500000*(batch-1); b = 500000*batch
batch = 7; c = 500000*(batch-1); d = 500000*batch
X_train_1f = np.concatenate([df_train.signal.values[a:b],df_train.signal.values[c:d]] ).reshape(( -1,1))
y_train_1f = np.concatenate([df_train.open_channels.values[a:b],df_train.open_channels.values[c:d]] ).reshape(( -1,1))
batch = 4; a = 500000*(batch-1); b = 500000*batch
batch = 8; c = 500000*(batch-1); d = 500000*batch
X_train_3 = np.concatenate([df_train.signal.values[a:b],df_train.signal.values[c:d]] ).reshape(( -1,1))
y_train_3 = np.concatenate([df_train.open_channels.values[a:b],df_train.open_channels.values[c:d]] ).reshape(( -1,1))
batch = 6; a = 500000*(batch-1); b = 500000*batch
batch = 9; c = 500000*(batch-1); d = 500000*batch
X_train_5 = np.concatenate([df_train.signal.values[a:b],df_train.signal.values[c:d]] ).reshape(( -1,1))
y_train_5 = np.concatenate([df_train.open_channels.values[a:b],df_train.open_channels.values[c:d]] ).reshape(( -1,1))
batch = 5; a = 500000*(batch-1); b = 500000*batch
batch = 10; c = 500000*(batch-1); d = 500000*batch
X_train_10 = np.concatenate([df_train.signal.values[a:b],df_train.signal.values[c:d]] ).reshape(( -1,1))
y_train_10 = np.concatenate([df_train.open_channels.values[a:b],df_train.open_channels.values[c:d]] ).reshape(( -1,1))
for X, y, num_classes, model_name, max_epoch in zip(
 [X_train_1s, X_train_1f, X_train_3, X_train_5, X_train_10],
[y_train_1s, y_train_1f, y_train_3, y_train_5, y_train_10],
np.array([1, 1, 3, 5, 10])+1,
['1s', '1f', '3', '5', '10'],
[30, 30, 60, 60, 120], 
):
X = X.reshape(-1, 1, time_step)
 y = y.reshape(-1, time_step)
 idx = np.arange(X.shape[0])
 kf = model_selection.KFold(5, shuffle=True, random_state=seed)
 train_idx, val_idx = list(kf.split(idx)) [fold]
print("model_name:", model_name)
 print("train dataset shape:", X[train_idx].shape, y[train_idx].shape)
 train_dataset = IonDataset(X[train_idx], y[train_idx], 'train', train_augment)
 valid_dataset = IonDataset(X[val_idx], y[val_idx], 'train', valid_augment)
 loaders = {
'train': DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4, pin_memory=True),
'valid': DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, num_workers=4, pin_memory=True),
}
model = Unet(num_classes=num_classes)
 loss_function = nn.CrossEntropyLoss()
optimizer = optim.AdamW(model.parameters() , lr=0.001, weight_decay=1e-5)
 scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=10, factor=0.1, min_lr=1e-8)
 metrics = [ClassificationMeter(num_classes),]
trainer = Trainer(device, workspace, model, optimizer, scheduler, loss_function, loaders, metrics,
model_name=model_name,
report_step_interval=-1, 
)
 trainer.train(max_epoch)
<feature_engineering>
SEED = 323
base_dir = '.. /input/'
def seed_everything(seed=SEED):
random.seed(seed)
 os.environ['PYHTONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(SEED )
if PREDICT:
start=500
a = 0; b = 100000
df_test.loc[df_test.index[a:b],'signal'] = df_test.signal.values[a:b] - 3*(df_test.time.values[a:b]-start)/10.
 start=510
a = 100000; b = 200000
df_test.loc[df_test.index[a:b],'signal'] = df_test.signal.values[a:b] - 3*(df_test.time.values[a:b]-start)/10.
 start=540
a = 400000; b = 500000
df_test.loc[df_test.index[a:b],'signal'] = df_test.signal.values[a:b] - 3*(df_test.time.values[a:b]-start)/10.
 start=560
a = 600000; b = 700000
df_test.loc[df_test.index[a:b],'signal'] = df_test.signal.values[a:b] - 3*(df_test.time.values[a:b]-start)/10.
 start=570
a = 700000; b = 800000
df_test.loc[df_test.index[a:b],'signal'] = df_test.signal.values[a:b] - 3*(df_test.time.values[a:b]-start)/10.
 start=580
a = 800000; b = 900000
df_test.loc[df_test.index[a:b],'signal'] = df_test.signal.values[a:b] - 3*(df_test.time.values[a:b]-start)/10.
 def f(x):
return -(0.00788)*(x-625)**2+2.345 +2.58
a = 1000000; b = 1500000
df_test.loc[df_test.index[a:b],'signal'] = df_test.signal.values[a:b] - f(df_test.time[a:b].values)
 sub = pd.read_csv(".. /input/liverpool-ion-switching/sample_submission.csv", dtype={'time':str})
 predictors = {}
for num_classes, model_name in zip(
 np.array([1, 1, 3, 5, 10])+1,
['1s', '1f', '3', '5', '10'] 
):
if not ENSEMBLE:
models = [Unet(num_classes=num_classes),]
predictor = Predictor(device, models)
 predictor.load_checkpoint(os.path.join(workspace, f"{model_name}_best.pth.tar"), 0)
 else:
models = [
Unet(num_classes=num_classes),
]
predictor = Predictor(device, models)
 predictor.load_checkpoint(os.path.join("workspace/unet-wide-4k-f0", f"{model_name}_best.pth.tar"), 0)
 predictors[model_name] = predictor
for start, end, model_type in zip(
 np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])*100000,
np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 20])*100000,
['1s', '3', '5', '1s', '1f', '10', '5', '10', '1s', '3', '1s'], 
):
X = df_test.signal.values[start:end].reshape(-1, 1, time_step_test)
 test_dataset = IonDataset(X, type='test')
 test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
 res = predictors[model_type].predict(test_loader)
 sub.iloc[start:end,1] = res
sub.to_csv("pred.csv", index=False)
 plt.figure(figsize=(20,5))
res = 1000
let = ['A','B','C','D','E','F','G','H','I','J']
plt.plot(range(0,df_test.shape[0],res),sub.open_channels[0::res])
 for i in range(5): plt.plot([i*500000,i*500000],[-5,12.5],'r')
 for i in range(21): plt.plot([i*100000,i*100000],[-5,12.5],'r:')
 for k in range(4): plt.text(k*500000+250000,10,str(k+1),size=20)
 for k in range(10): plt.text(k*100000+40000,7.5,let[k],size=16)
 plt.title('Test Data Predictions',size=16)
 plt.show() 
<set_options>
labels = pd.read_csv(base_dir+'train_labels.csv' )
py.init_notebook_mode(connected=True)
<set_options>
labels.label.value_counts()
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df<load_from_disk>
tr, val = train_test_split(labels.label, stratify=labels.label, test_size=0.101, random_state=SEED )
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 test_df = pd.read_feather(root/'test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )<feature_engineering>
img_class_dict = {k:v for k, v in zip(labels.id, labels.label)}
leak_df = pd.read_feather('.. /input/ashrae-leak-data-station/leak.feather')
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[(leak_df.timestamp.dt.year > 2016)&(leak_df.timestamp.dt.year < 2019)]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
leak_df = leak_df[leak_df.building_id!=245]<count_values>
class CancerDataset(Dataset):
def __init__(self, datafolder, datatype='train', transform = transforms.Compose([transforms.CenterCrop(64),transforms.ToTensor() ]), labels_dict={}):
self.datafolder = datafolder
self.datatype = datatype
self.image_files_list = [s for s in os.listdir(datafolder)]
self.transform = transform
self.labels_dict = labels_dict
if self.datatype == 'train':
self.labels = [labels_dict[i.split('.')[0]] for i in self.image_files_list]
else:
self.labels = [0 for _ in range(len(self.image_files_list)) ]
def __len__(self):
return len(self.image_files_list)
 def __getitem__(self, idx):
img_name = os.path.join(self.datafolder, self.image_files_list[idx])
 img = cv2.imread(img_name)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 image = self.transform(image=img)
 image = image['image']
img_name_short = self.image_files_list[idx].split('.')[0]
if self.datatype == 'train':
label = self.labels_dict[img_name_short]
else:
label = 0
return image, label
leak_df.meter.value_counts()<count_duplicates>
data_transforms = albumentations.Compose([
albumentations.Resize(224, 224),
albumentations.RandomRotate90(p=0.5),
albumentations.Transpose(p=0.5),
albumentations.Flip(p=0.5),
albumentations.OneOf([
albumentations.CLAHE(clip_limit=2), albumentations.IAASharpen() , albumentations.IAAEmboss() ,
albumentations.RandomBrightness() , albumentations.RandomContrast() ,
albumentations.JpegCompression() , albumentations.Blur() , albumentations.GaussNoise() ], p=0.5),
albumentations.HueSaturationValue(p=0.5),
albumentations.ShiftScaleRotate(shift_limit=0.15, scale_limit=0.15, rotate_limit=45, p=0.5),
albumentations.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225]),
AT.ToTensor()
])
 data_transforms_test = albumentations.Compose([
albumentations.Resize(224, 224),
albumentations.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225]),
AT.ToTensor()
])
 data_transforms_tta0 = albumentations.Compose([
albumentations.Resize(224, 224),
albumentations.RandomRotate90(p=0.5),
albumentations.Transpose(p=0.5),
albumentations.Flip(p=0.5),
albumentations.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225]),
AT.ToTensor()
])
 data_transforms_tta1 = albumentations.Compose([
albumentations.Resize(224, 224),
albumentations.RandomRotate90(p=1),
albumentations.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225]),
AT.ToTensor()
])
 data_transforms_tta2 = albumentations.Compose([
albumentations.Resize(224, 224),
albumentations.Transpose(p=1),
albumentations.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225]),
AT.ToTensor()
])
 data_transforms_tta3 = albumentations.Compose([
albumentations.Resize(224, 224),
albumentations.Flip(p=1),
albumentations.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225]),
AT.ToTensor()
])
 dataset = CancerDataset(datafolder=base_dir+'train/', datatype='train', transform=data_transforms, labels_dict=img_class_dict)
 val_set = CancerDataset(datafolder=base_dir+'train/', datatype='train', transform=data_transforms_test, labels_dict=img_class_dict)
 test_set = CancerDataset(datafolder=base_dir+'test/', datatype='test', transform=data_transforms_test)
 train_sampler = SubsetRandomSampler(list(tr.index))
valid_sampler = SubsetRandomSampler(list(val.index))
batch_size = 16
num_workers = 0
train_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=train_sampler, num_workers=num_workers)
 valid_loader = torch.utils.data.DataLoader(val_set, batch_size=batch_size, sampler=valid_sampler, num_workers=num_workers)
 test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers )
print(leak_df.duplicated().sum() )<feature_engineering>
model_conv = ptcv_get_model("cbam_resnet50", pretrained=True)
 model_conv.avg_pool = nn.AdaptiveAvgPool2d(( 1, 1))
model_conv.last_linear = nn.Sequential(nn.Dropout(0.6), nn.Linear(in_features=2048, out_features=512, bias=True), nn.SELU() ,
nn.Dropout(0.8), nn.Linear(in_features=512, out_features=1, bias=True))
print(len(leak_df)/ len(train_df))<set_options>
model_conv.cuda()
criterion = nn.BCEWithLogitsLoss()
optimizer = optim.Adam(model_conv.parameters() , lr=0.0004)
 scheduler = StepLR(optimizer, 5, gamma=0.2)
 scheduler.step()
del train_df
gc.collect()<load_from_csv>
val_auc_max = 0
patience = 25
p = 0
stop = False
n_epochs = 3
for epoch in range(1, n_epochs+1):
if stop:
print("Training stop.")
 break
print(time.ctime() , 'Epoch:', epoch)
 train_loss = []
train_auc = []
for tr_batch_i,(data, target)in enumerate(train_loader):
model_conv.train()
data, target = data.cuda() , target.cuda()
optimizer.zero_grad()
output = model_conv(data)
 loss = criterion(output[:,0], target.float())
 train_loss.append(loss.item())
 a = target.data.cpu().numpy()
try:
b = output[:,0].detach().cpu().numpy()
train_auc.append(roc_auc_score(a, b))
except:
pass
loss.backward()
optimizer.step()
if(tr_batch_i+1)%600 == 0:
model_conv.eval()
val_loss = []
val_auc = []
for val_batch_i,(data, target)in enumerate(valid_loader):
data, target = data.cuda() , target.cuda()
output = model_conv(data)
 loss = criterion(output[:,0], target.float())
 val_loss.append(loss.item())
 a = target.data.cpu().numpy()
try:
b = output[:,0].detach().cpu().numpy()
val_auc.append(roc_auc_score(a, b))
except:
pass
print('Epoch %d, batches:%d, train loss: %.4f, valid loss: %.4f.'%(epoch, tr_batch_i, np.mean(train_loss), np.mean(val_loss))
+ ' train auc: %.4f, valid auc: %.4f'%(np.mean(train_auc),np.mean(val_auc)))
 train_loss = []
train_auc = []
valid_auc = np.mean(val_auc)
 if valid_auc > val_auc_max:
print('Validation auc increased({:.6f} --> {:.6f} ).Saving model...'.format(
 val_auc_max,
valid_auc))
torch.save(model_conv.state_dict() , 'model.pt')
 val_auc_max = valid_auc
p = 0
else:
p += 1
if p > patience:
print('Early stop training')
 stop = True
break
scheduler.step()
sample_submission1 = pd.read_csv('.. /input/ashrae-kfold-lightgbm-without-leak-1-08/submission.csv', index_col=0)
 sample_submission2 = pd.read_csv('.. /input/ashrae-half-and-half/submission.csv', index_col=0)
 sample_submission3 = pd.read_csv('.. /input/ashrae-highway-kernel-route4/submission.csv', index_col=0)
 sample_submission4 = pd.read_csv('.. /input/stratifiedkfoldlgbxopy/submission.csv', index_col=0 )<feature_engineering>
torch.cuda.empty_cache()
test_df['pred1'] = sample_submission1.meter_reading
test_df['pred2'] = sample_submission2.meter_reading
test_df['pred3'] = sample_submission3.meter_reading
test_df['pred4'] = sample_submission4.meter_reading
test_df.loc[test_df.pred3<0, 'pred3'] = 0
test_df.loc[test_df.pred3<0, 'pred4'] = 0
del sample_submission1, sample_submission2, sample_submission3, sample_submission4
gc.collect()
test_df = reduce_mem_usage(test_df)
 leak_df = reduce_mem_usage(leak_df )<merge>
model_conv.eval()
leak_df = leak_df.merge(test_df[['building_id', 'meter', 'timestamp', 'pred1', 'pred2', 'pred3', 'pred4', 'row_id']], left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 leak_df = leak_df.merge(building_meta_df[['building_id', 'site_id']], on='building_id', how='left' )<feature_engineering>
NUM_TTA = 32
leak_df['pred1_l1p'] = np.log1p(leak_df.pred1)
 leak_df['pred2_l1p'] = np.log1p(leak_df.pred2)
 leak_df['pred3_l1p'] = np.log1p(leak_df.pred3)
 leak_df['pred4_l1p'] = np.log1p(leak_df.pred4)
 leak_df['meter_reading_l1p'] = np.log1p(leak_df.meter_reading )<filter>
sigmoid = lambda x: scipy.special.expit(x )
leak_df[leak_df.pred1_l1p.isnull() ]<compute_test_metric>
for num_tta in range(NUM_TTA):
if num_tta==0:
test_set = CancerDataset(datafolder=base_dir+'test/', datatype='test', transform=data_transforms_test)
 test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)
 elif num_tta==1:
test_set = CancerDataset(datafolder=base_dir+'test/', datatype='test', transform=data_transforms_tta1)
 test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)
 elif num_tta==2:
test_set = CancerDataset(datafolder=base_dir+'test/', datatype='test', transform=data_transforms_tta2)
 test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)
 elif num_tta==3:
test_set = CancerDataset(datafolder=base_dir+'test/', datatype='test', transform=data_transforms_tta3)
 test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)
 elif num_tta<8:
test_set = CancerDataset(datafolder=base_dir+'test/', datatype='test', transform=data_transforms_tta0)
 test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)
 else:
test_set = CancerDataset(datafolder=base_dir+'test/', datatype='test', transform=data_transforms)
 test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)
 preds = []
for batch_i,(data, target)in enumerate(test_loader):
data, target = data.cuda() , target.cuda()
output = model_conv(data ).detach()
pr = output[:,0].cpu().numpy()
for i in pr:
preds.append(sigmoid(i)/NUM_TTA)
 if num_tta==0:
test_preds = pd.DataFrame({'imgs': test_set.image_files_list, 'preds': preds})
 test_preds['imgs'] = test_preds['imgs'].apply(lambda x: x.split('.')[0])
 else:
test_preds['preds']+=np.array(preds)
 print(num_tta)
 sub = pd.read_csv('.. /input/sample_submission.csv')
 sub = pd.merge(sub, test_preds, left_on='id', right_on='imgs')
 sub = sub[['id', 'preds']]
sub.columns = ['id', 'label']
sub.to_csv('sub_tta.csv', index=False )
<define_search_model><EOS> 
print("Finished" )