Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<load_from_csv>
%matplotlib inline
np.random.seed(42 )
def read_data() :
train = pd.read_csv('/kaggle/input/clean-kalman/train_clean_kalman.csv', dtype={'time': np.float32, 'signal': np.float32, 'open_channels':np.int32})
 test = pd.read_csv('/kaggle/input/clean-kalman/test_clean_kalman.csv', dtype={'time': np.float32, 'signal': np.float32})
 Y_train_proba = np.load("/kaggle/input/ion-shifted-rfc-proba/Y_train_proba.npy")
 Y_test_proba = np.load("/kaggle/input/ion-shifted-rfc-proba/Y_test_proba.npy")
 for i in range(11):
train[f"proba_{i}"] = Y_train_proba[:, i]
test[f"proba_{i}"] = Y_test_proba[:, i]
sub = pd.read_csv('/kaggle/input/liverpool-ion-switching/sample_submission.csv', dtype={'time': np.float32})
 return train, test, sub
def batching(df, batch_size):
df['group'] = df.groupby(df.index//batch_size, sort=False)['signal'].agg(['ngroup'] ).values
df['group'] = df['group'].astype(np.uint16)
 return df
def normalize(train, test):
train_input_mean = train.signal.mean()
train_input_sigma = train.signal.std()
train['signal'] =(train.signal - train_input_mean)/ train_input_sigma
test['signal'] =(test.signal - train_input_mean)/ train_input_sigma
return train, test
def lag_with_pct_change(df, windows):
for window in windows:
df['signal_shift_pos_' + str(window)] = df.groupby('group')['signal'].shift(window ).fillna(0)
 df['signal_shift_neg_' + str(window)] = df.groupby('group')['signal'].shift(-1 * window ).fillna(0)
 return df
def run_feat_engineering(df, batch_size):
df = batching(df, batch_size = batch_size)
 df = lag_with_pct_change(df, [1, 2, 3])
 df['signal_2'] = df['signal'] ** 2
return df
def feature_selection(train, test):
features = [col for col in train.columns if col not in ['index', 'group', 'open_channels', 'time']]
train = train.replace([np.inf, -np.inf], np.nan)
 test = test.replace([np.inf, -np.inf], np.nan)
 for feature in features:
feature_mean = pd.concat([train[feature], test[feature]], axis = 0 ).mean()
train[feature] = train[feature].fillna(feature_mean)
 test[feature] = test[feature].fillna(feature_mean)
 return train, test, features
def split(GROUP_BATCH_SIZE=4000, SPLITS=5):
print('Reading Data Started...')
 train, test, sample_submission = read_data()
train, test = normalize(train, test)
 print('Reading and Normalizing Data Completed')
 print('Creating Features')
 print('Feature Engineering Started...')
 train = run_feat_engineering(train, batch_size=GROUP_BATCH_SIZE)
 test = run_feat_engineering(test, batch_size=GROUP_BATCH_SIZE)
 train, test, features = feature_selection(train, test)
 print(train.head())
 print('Feature Engineering Completed...')
 target = ['open_channels']
group = train['group']
kf = GroupKFold(n_splits=SPLITS)
 splits = [x for x in kf.split(train, train[target], group)]
new_splits = []
for sp in splits:
new_split = []
new_split.append(np.unique(group[sp[0]]))
new_split.append(np.unique(group[sp[1]]))
new_split.append(sp[1])
 new_splits.append(new_split)
 target_cols = ['open_channels']
print(train.head() , train.shape)
 train_tr = np.array(list(train.groupby('group' ).apply(lambda x: x[target_cols].values)) ).astype(np.float32)
 train = np.array(list(train.groupby('group' ).apply(lambda x: x[features].values)))
 test = np.array(list(test.groupby('group' ).apply(lambda x: x[features].values)))
 print(train.shape, test.shape, train_tr.shape)
 return train, test, train_tr, new_splits<normalization>
DATA_DIR = '.. /input'
train_dir = os.path.join(DATA_DIR, 'train')
 test_dir = os.path.join(DATA_DIR, 'test')
 def train_validation_split(df, val_fraction=0.1):
val_ids = np.random.choice(df.id, size=int(len(df)* val_fraction))
val_df = df.query('id in @val_ids')
 train_df = df.query('id not in @val_ids')
 return train_df, val_df
train_label_df, val_label_df = train_validation_split(pd.read_csv(os.path.join(DATA_DIR, 'train_labels.csv')) ,
val_fraction=0.1 )
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )<import_modules>
def function_timer(function):
def wrapper(*args, **kwargs):
start = time.time()
result = function(*args, **kwargs)
 duration = time.time() - start
hours = int(duration // 60**2)
 minutes = int(( duration % 60**2)// 60)
 seconds = int(duration % 60)
 print(f'execution-time of function "{function.__name__}": {hours}h {minutes}m {seconds}s')
 return result
return wrapper
class HistoPatches(Dataset):
def __init__(self,
image_dir: str,
label_df=None,
transform=transforms.ToTensor() ,
sample_n=None,
in_memory=False):

 self.image_dir = image_dir
self.label_df = label_df
self.transform = transform
self.in_memory = in_memory
if label_df is not None:
if sample_n:
self.label_df = self.label_df.sample(n=sample_n)
 ids = set(self.label_df.id)
 self.img_files = [f for f in os.listdir(image_dir)if f.split('.')[0] in ids]
else:
if sample_n is not None:
print('subsampling is currently only implemented when a label-dataframe is provided.')
 return
self.img_files = os.listdir(image_dir)
 if in_memory:
self.id2image = self._load_images()
print(f'Initialized datatset with {len(self.img_files)} images.
 ')
 @function_timer
def _load_images(self):
print('loading images in memory...')
 id2image = {}
for file_name in self.img_files:
img = PIL.Image.open(os.path.join(self.image_dir, file_name))
X = self.transform(img)
 id_ = file_name.split('.')[0]
id2image[id_] = X
return id2image
def __getitem__(self, idx):
file_name = self.img_files[idx]
id_ = file_name.split('.')[0]
if self.in_memory:
X = self.id2image[id_]
else:
img = PIL.Image.open(os.path.join(self.image_dir, file_name))
X = self.transform(img)
 if self.label_df is not None:
y = float(self.label_df.query('id == @id_' ).label)
 return X, y
else:
return X, id_
def __len__(self):
return len(self.img_files)
 memory = False
batchsize = 64
image_trans = transforms.Compose([
transforms.ToTensor() ,
transforms.Normalize(mean=[0.70017236, 0.5436771, 0.6961061],
std=[0.22246036, 0.26757348, 0.19798167])
 ])
 train_trans = transforms.Compose([transforms.ToTensor() ,
transforms.Normalize(mean=[0.70017236, 0.5436771, 0.6961061],
std=[0.22246036, 0.26757348, 0.19798167]),
transforms.RandomRotation(( -180, 180)) ,
transforms.RandomHorizontalFlip() ])
 train = HistoPatches(train_dir,
train_label_df,
transform=image_trans,
in_memory=memory)
 val = HistoPatches(train_dir,
val_label_df,
transform=image_trans,
in_memory=memory)
 train_loader = DataLoader(train, batch_size=batchsize, shuffle=True)
 val_loader = DataLoader(val, batch_size=batchsize, shuffle=False)
 X, y = next(iter(train_loader))
print('batch-dimensions: ', X.shape, y.shape )
import torch.nn.functional as F 
<define_search_model>
def conv_dim(in_dim, k=3, s=1, p=0, p_left=None, p_right=None):

 if p is not None:
p_left = p_right = p
assert p_left is not None and p_right is not None
tmp =(in_dim - k + p_left + p_right)/ s
out_dim = int(np.floor(tmp)+ 1)
 if tmp % 1 != 0:
print('no exact output-dim; using Gauss-brackets.')
 print(f'out-dim: {out_dim}')
 conv_dim(30, k=3, s=1, p=0 )
class Wave_Block(nn.Module):
def __init__(self,in_channels,out_channels,dilation_rates):
super(Wave_Block,self ).__init__()
self.num_rates = dilation_rates
self.convs = nn.ModuleList()
self.filter_convs = nn.ModuleList()
self.gate_convs = nn.ModuleList()
self.convs.append(nn.Conv1d(in_channels,out_channels,kernel_size=1))
dilation_rates = [2**i for i in range(dilation_rates)]
for dilation_rate in dilation_rates:
self.filter_convs.append(nn.Conv1d(out_channels,out_channels,kernel_size=3,padding=dilation_rate,dilation=dilation_rate))
self.gate_convs.append(nn.Conv1d(out_channels,out_channels,kernel_size=3,padding=dilation_rate,dilation=dilation_rate))
self.convs.append(nn.Conv1d(out_channels,out_channels,kernel_size=1))
def forward(self,x):
x = self.convs[0](x)
 res = x
for i in range(self.num_rates):
x = F.tanh(self.filter_convs[i](x)) *F.sigmoid(self.gate_convs[i](x))
x = self.convs[i+1](x)
 res = torch.add(res, x)
 return res
class Classifier(nn.Module):
def __init__(self):
super().__init__()
input_size = 128
self.LSTM1 = nn.GRU(input_size=19,hidden_size=64,num_layers=2,batch_first=True,bidirectional=True)
 self.LSTM = nn.GRU(input_size=input_size,hidden_size=64,num_layers=2,batch_first=True,bidirectional=True)
 self.wave_block1 = Wave_Block(128,16,12)
 self.wave_block2 = Wave_Block(16,32,8)
 self.wave_block3 = Wave_Block(32,64,4)
 self.wave_block4 = Wave_Block(64, 128, 1)
 self.fc = nn.Linear(128, 11)
 def forward(self,x):
x,_ = self.LSTM1(x)
 x = x.permute(0, 2, 1)
 x = self.wave_block1(x)
 x = self.wave_block2(x)
 x = self.wave_block3(x)
 x = self.wave_block4(x)
 x = x.permute(0, 2, 1)
 x,_ = self.LSTM(x)
 x = self.fc(x)
 return x
class EarlyStopping:
def __init__(self, patience=7, delta=0, checkpoint_path='checkpoint.pt', is_maximize=True):
self.patience, self.delta, self.checkpoint_path = patience, delta, checkpoint_path
self.counter, self.best_score = 0, None
self.is_maximize = is_maximize
def load_best_weights(self, model):
model.load_state_dict(torch.load(self.checkpoint_path))
def __call__(self, score, model):
if self.best_score is None or \ 
(score > self.best_score + self.delta if self.is_maximize else score < self.best_score - self.delta):
torch.save(model.state_dict() , self.checkpoint_path)
 self.best_score, self.counter = score, 0
return 1
else:
self.counter += 1
if self.counter >= self.patience:
return 2
return 0<train_model>
net = models.densenet121(pretrained=False)
 net.features = nn.Sequential(
 nn.Sequential(OrderedDict([ 
('conv0', nn.Conv2d(3, 64, kernel_size=3, stride=2, padding=1, bias=False)) , 
('norm0', nn.BatchNorm2d(64)) , 
('relu0', nn.ReLU(inplace=True))
])) ,
net.features[4:])
 net.classifier = nn.Sequential(
 nn.Linear(1024, 512),
nn.Dropout(p=0.1),
nn.ReLU() ,
nn.Linear(512, 1)
 )
class IronDataset(Dataset):
def __init__(self, data, labels, training=True, transform=None, seq_len=5000, flip=0.5, noise_level=0, class_split=0.0):
self.data = data
self.labels = labels
self.transform = transform
self.training = training
self.flip = flip
self.noise_level = noise_level
self.class_split = class_split
self.seq_len = seq_len
def __len__(self):
return len(self.data)
 def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
data = self.data[idx]
labels = self.labels[idx]
return [data.astype(np.float32), labels.astype(int)]<split>
myloss = nn.BCEWithLogitsLoss()
net.cuda()
X, y = next(iter(train_loader))
X , y = X.cuda() , y.cuda()
print('X, y shapes: ', X.shape, y.shape)
 out = net(X ).squeeze()
print(f'loss: {myloss(y, out.type(torch.DoubleTensor ).cuda() ).item() }' )
train, test, train_tr, new_splits = split()<install_modules>
@function_timer
def train_model(net, train, validation, optimizer, device, max_epoch=100, verbose=False):

 global error_df
error_df = pd.DataFrame(columns=['train_bce', 'train_acc', 'train_auc', 'val_bce', 'val_acc', 'val_auc'])
 criterion = nn.BCEWithLogitsLoss()
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9)
 net.to(device)
 print('epoch\tLR\ttr-BCE\ttr-Acc\ttr-AUC\t\tval-BCE\tval-Acc\tval-AUC')
 for epoch in range(max_epoch):
net.train()
training_bce = training_acc = training_auc = 0
for X, y in train:
X , y = X.to(device), y.to(device)
 optimizer.zero_grad()
out = net(X ).squeeze()
labels = y.detach().cpu().numpy()
probabilities = torch.sigmoid(out ).detach().cpu().numpy()
predictions = probabilities.round()
loss = criterion(out.type(torch.DoubleTensor ).cuda() , y)
 training_bce += loss.item()
training_acc += np.mean(labels == predictions)* 100
training_auc += roc_auc_score(y_true=labels, y_score=probabilities)
 loss.backward()
optimizer.step()
with torch.no_grad() :
net.eval()
validation_bce = validation_acc = validation_auc = 0
for X, y in validation:
X , y = X.to(device), y.to(device)
 out = net(X ).squeeze()
labels = y.detach().cpu().numpy()
probabilities = torch.sigmoid(out ).detach().cpu().numpy()
predictions = probabilities.round()
validation_bce += criterion(out.type(torch.DoubleTensor ).cuda() , y ).item()
validation_acc += np.mean(labels == predictions)* 100
validation_auc += roc_auc_score(y_true=labels, y_score=probabilities)
 training_bce /= len(train)
 training_acc /= len(train)
 training_auc /= len(train)
 validation_bce /= len(validation)
 validation_acc /= len(validation)
 validation_auc /= len(validation)
 scheduler.step()
error_stats = [training_bce, training_acc, training_auc, validation_bce, validation_acc, validation_auc]
error_df = error_df.append(pd.Series(error_stats, index=error_df.columns), ignore_index=True)
 print('{}\t{:.4f}\t{:.4f}\t{:.2f}\t{:.4f}\t\t{:.4f}\t{:.2f}\t{:.4f}'.format(epoch, optimizer.param_groups[0]['lr'], *error_stats))
pip install torchcontrib<import_modules>
optimizer = torch.optim.Adam(net.parameters() , lr=0.01, weight_decay=0)
 train_model(net,
train_loader,
val_loader,
optimizer,
device=torch.device('cuda:0' if torch.cuda.is_available() else 'cpu'),
max_epoch=9,
verbose=False)
 torch.save(net.state_dict() , 'swag_net.pt' )
from torchcontrib.optim import SWA
import torchcontrib<choose_model_class>
def predict_and_display(net, testset, n=10):

 net.eval()
for i in choice(range(len(testset)) , size=n):
image, label = test_set[i]
output = net(image.unsqueeze(0 ).cuda())
 prob, pred = prediction_from_output(output.unsqueeze(0))
prob, pred = prob.item() , pred.item()
evaluation = 'correct' if pred == label else 'mistake'
plt.figure(figsize=(2, 2))
print(f'
truth: {label} | pred: {pred} | prob: {prob:.2f}')
 print(f'{evaluation}:({class_name(label)} vs.{class_name(pred)})')
 show_image(image, means=channel_means, stdevs=channel_standard_devs)
 def plot_error_curves(training_error, validation_error, error_name='error', ylim=None, save_fig=True):

 assert len(training_error)== len(validation_error)> 1
fig, ax = plt.subplots()
ax.plot(range(len(training_error)) , training_error)
 ax.plot(range(len(validation_error)) , validation_error)
 if ylim:
ax.set_ylim(*ylim)
 ax.set_xlabel('epoch')
 ax.set_ylabel('CE')
 ax.legend(( 'training', 'validation'))
ax.set_title(f'{error_name} over time')
 if save_fig:
fig.savefig(f'{error_name.replace(" ", "_")}_learning_curve', bbox_inches='tight', transparent=True)
 plt.show() ;
<create_dataframe><EOS> 
def write_submission_file() :
test = HistoPatches(image_dir=os.path.join(DATA_DIR, 'test'), transform=image_trans)
 test_loader = DataLoader(test, batch_size=16)
 prediction_df = pd.DataFrame(columns=['id', 'label'])
 for i,(X, ids)in enumerate(test_loader):
out = net(X.cuda() ).squeeze()
probabilities = torch.sigmoid(out ).detach().cpu().numpy()
df = pd.DataFrame({'id': ids, 'label': probabilities})
 prediction_df = prediction_df.append(df)
 display(prediction_df.head())
 prediction_df.to_csv('submission.csv', index=False)
 os.listdir('.')
 write_submission_file()