Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<split>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.2 )<import_modules>
from fastai import *
from fastai.vision import *
from torchvision.models import *
import os
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense,Flatten,BatchNormalization,Dropout,Conv2D,MaxPool2D<set_options>
path = Path(".. /input")
 labels = pd.read_csv(path/"train_labels.csv")
 labels.head()
print(tf.config.list_physical_devices('GPU'),'//',tf.test.is_built_with_cuda() )<feature_engineering>
print(labels["label"].nunique()); classes = list(set(labels["label"])) ; classes
datagen = tf.keras.preprocessing.image.ImageDataGenerator(
 rotation_range=12,
width_shift_range=0.12,
height_shift_range=0.12,
shear_range=0.12,
validation_split=0.2
)<train_model>
for i in classes:
print("Number of items in class {} is {}".format(i,len(labels[labels["label"] == i])) )
training_generator = datagen.flow(X_train, y_train, batch_size=32,subset='training')
 validation_generator = datagen.flow(X_train, y_train, batch_size=32,subset='validation' )<choose_model_class>
tfms = get_transforms(do_flip = True,flip_vert = True,max_zoom = 1.1 )
model=Sequential()<choose_model_class>
np.random.seed(123)
 sz = 32
data = ImageDataBunch.from_csv(path, folder = 'train', csv_labels = "train_labels.csv",
test = 'test',suffix=".tif", size = sz,bs = 256,
ds_tfms = tfms)
 data.path = pathlib.Path('.')
 data.normalize(imagenet_stats )
model.add(Conv2D(filters = 32, kernel_size =(3,3),padding = 'Same',activation ='relu', input_shape =(28,28,1)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters = 32, kernel_size =(3,3),padding = 'Same',activation ='relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.3))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',activation ='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',activation ='relu'))
model.add(BatchNormalization())
 model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.3))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(BatchNormalization())
 model.add(Dropout(0.3))
model.add(Dense(256, activation = "relu"))
model.add(BatchNormalization())
 model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))<choose_model_class>
data.show_batch(rows = 3 )
model.compile(
 loss="sparse_categorical_crossentropy",
optimizer=tf.keras.optimizers.RMSprop(lr=0.003, rho=0.9, epsilon=1e-08, decay=0.0),
metrics=["accuracy"]
)<choose_model_class>
def auc_score(y_pred,y_true,tens=True):
score = roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])
 if tens:
score = tensor(score)
 return score
cb=tf.keras.callbacks.EarlyStopping(patience=10,restore_best_weights=True)
 learning_rate_reduction = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_accuracy',
patience=3,
verbose=1,
factor=0.5,
min_lr=0.00001 )<train_model>
arch = models.densenet121
learn = cnn_learner(data,arch,pretrained = True,ps = 0.45,metrics = [auc_score,accuracy] )
model.fit(training_generator,epochs=100,callbacks=[cb,learning_rate_reduction],validation_data=validation_generator )<compute_test_metric>
learn.fit_one_cycle(8,1e-2 )
model.evaluate(X_test, y_test )<load_from_csv>
learn.unfreeze()
learn.lr_find()
learn.recorder.plot()
pred_Data=np.array(pd.read_csv('.. /input/digit-recognizer/test.csv')/ 255.)
X_pred=pred_Data.reshape(( -1,28,28,1))<predict_on_test>
learn.fit_one_cycle(2,max_lr = slice(1e-5,1e-3))
predictions=model.predict_classes(X_pred )<save_to_csv>
newTfms = get_transforms(do_flip = True,flip_vert = True,max_zoom = 1.25)
 newSz = 64
newData = ImageDataBunch.from_csv(path, folder = 'train', csv_labels = "train_labels.csv",
test = 'test',suffix=".tif", size = newSz, ds_tfms = newTfms)
 newData.path = pathlib.Path('.')
 newData.normalize(imagenet_stats)
 learn.data = newData
submit=pd.DataFrame({'ImageId':range(1,len(predictions)+1),'Label':predictions})
 submit.to_csv('submission.csv',index=False )<set_options>
learn.freeze()
learn.lr_find()
learn.recorder.plot()
warnings.filterwarnings('ignore')
 sns.set_context("paper", font_scale = 1, rc={"grid.linewidth": 3})
 pd.set_option('display.max_rows', 100, 'display.max_columns', 400)
<load_from_csv>
learn.fit_one_cycle(8,1e-2 )
train_data=pd.read_csv('.. /input/digit-recognizer/train.csv')
 test_data=pd.read_csv('.. /input/digit-recognizer/test.csv')
 sample_data = pd.read_csv('.. /input/digit-recognizer/sample_submission.csv' )<prepare_x_and_y>
learn.unfreeze()
learn.lr_find()
learn.recorder.plot()
train_df = train_data.iloc[:, 1:].values
y_train = train_data.iloc[:, 0].values
test_df = test_data.values<data_type_conversions>
learn.fit_one_cycle(2,max_lr = slice(1e-3/3,1e-3))
img_tform_1 = transforms.Compose([
transforms.ToPILImage() ,transforms.ToTensor() ,transforms.Normalize(( 0.5),(0.5)) ])
 img_tform_2 = transforms.Compose([
transforms.ToPILImage() ,transforms.RandomRotation(10),transforms.ToTensor() ,transforms.Normalize(( 0.5),(0.5)) ])
 img_tform_3 = transforms.Compose([
transforms.ToPILImage() ,transforms.RandomRotation(20),transforms.ToTensor() ,transforms.Normalize(( 0.5),(0.5)) ])
 img_tform_4 = transforms.Compose([
transforms.ToPILImage() ,transforms.RandomAffine(degrees=15, translate=(0.1,0.1), scale=(0.85,0.85)) ,\
transforms.ToTensor() ,transforms.Normalize(( 0.5),(0.5)) ])
 img_tform_5 = transforms.Compose([
transforms.ToPILImage() ,transforms.RandomAffine(0,shear=30,scale=[1.15,1.15]),\
transforms.ToTensor() ,transforms.Normalize(( 0.5),(0.5)) ])
 img_tform_6 = transforms.Compose([
transforms.ToPILImage() ,transforms.RandomAffine(0,shear=20,scale=[0.8,0.8]),\
transforms.ToTensor() ,transforms.Normalize(( 0.5),(0.5)) ])
 img_tform_7 = transforms.Compose([
transforms.ToPILImage() ,transforms.RandomAffine(degrees=30, scale=(1.2,1.2)) ,\
transforms.ToTensor() ,transforms.Normalize(( 0.5),(0.5)) ] )<data_type_conversions>
learn.save('stage-2' )
class MnistDataset(Dataset):
def __init__(self, features,transform=img_tform_1):
self.features = features.iloc[:,1:].values.reshape(( -1,28,28)).astype(np.uint8)
 self.targets = torch.from_numpy(features.label.values)
 self.transform=transform
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
return self.transform(self.features[idx]),self.targets[idx]
class TestDataset(Dataset):
def __init__(self, features,transform=img_tform_1):
self.features = features.values.reshape(( -1,28,28)).astype(np.uint8)
 self.targets = None
self.transform=transform
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
return self.transform(self.features[idx] )<split>
preds,y = learn.TTA()
acc = accuracy(preds, y)
 print('The validation accuracy is {} %.'.format(acc * 100))
pred_score = auc_score(preds,y ).item()
print('The validation AUC is {}.'.format(pred_score))
def create_dataloaders(seed, test_size=0.1, df=train_data, batch_size=32):
train_df, val_df = train_test_split(df,test_size=test_size,random_state=seed)
 train_data_1 = MnistDataset(train_df)
 train_data_2 = MnistDataset(train_df, img_tform_2)
 train_data_3 = MnistDataset(train_df, img_tform_3)
 train_data_4 = MnistDataset(train_df, img_tform_4)
 train_data_5 = MnistDataset(train_df, img_tform_5)
 train_data_6 = MnistDataset(train_df, img_tform_6)
 train_data_7 = MnistDataset(train_df, img_tform_7)
 train_final = ConcatDataset([train_data_1, train_data_2, train_data_3, train_data_4, train_data_5,\
train_data_6,train_data_7])
 val_data = MnistDataset(val_df)
 train_loader = torch.utils.data.DataLoader(train_final, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(val_data, batch_size=batch_size, shuffle=False)
 return train_loader, valid_loader<choose_model_class>
newTfms = get_transforms(do_flip = True,flip_vert = True,max_zoom = 1.5)
 newSz = 96
newData = ImageDataBunch.from_csv(path, folder = 'train', csv_labels = "train_labels.csv",
test = 'test',suffix=".tif", size = newSz, ds_tfms = newTfms)
 newData.path = pathlib.Path('.')
 newData.normalize(imagenet_stats)
 learn.data = newData
class Model(nn.Module):
def __init__(self):
super().__init__()
self.conv1 = nn.Sequential(
 nn.Conv2d(1, 32, kernel_size=3),
nn.BatchNorm2d(32),
nn.LeakyReLU(inplace=True),
nn.Conv2d(32, 32, kernel_size=3),
nn.BatchNorm2d(32),
nn.LeakyReLU(inplace=True),
nn.Conv2d(32, 32, kernel_size=5, stride=2, padding=14),
nn.BatchNorm2d(32),
nn.LeakyReLU(inplace=True),
nn.MaxPool2d(2, 2),
nn.Dropout2d(0.25),
nn.Conv2d(32, 64, kernel_size=3),
nn.BatchNorm2d(64),
nn.LeakyReLU(inplace=True),
nn.Conv2d(64, 64, kernel_size=3),
nn.BatchNorm2d(64),
nn.LeakyReLU(inplace=True),
nn.Conv2d(64, 64, kernel_size=5, stride=2, padding=6),
nn.BatchNorm2d(64),
nn.LeakyReLU(inplace=True),
nn.MaxPool2d(2, 2),
nn.Dropout2d(0.25),
nn.Conv2d(64, 128, kernel_size=4),
nn.BatchNorm2d(128),
nn.LeakyReLU(inplace=True),
nn.Dropout2d(0.25)
)
 self.fc = nn.Sequential(
 nn.Linear(128*1*1, 10)
)
 def forward(self, x):
x = self.conv1(x)
 x = x.view(-1, 128*1*1)
 x = self.fc(x)
 return x<train_model>
learn.freeze()
learn.lr_find()
learn.recorder.plot()
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
train_acc=0
total=0
train_preds=[]
for features,labels in dataloader:
optimizer.zero_grad()
inputs, targets = features.to(device), labels.to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
total+=len(targets)
 final_loss += loss.item()
train_preds.append(outputs.sigmoid().detach().cpu().numpy())
 _, predicted = torch.max(outputs, 1)
 train_acc+=(( predicted == targets ).sum().item())
 final_loss /= len(dataloader)
 train_preds = np.concatenate(train_preds)
 train_acc=(train_acc/total)*100
return final_loss,train_acc
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
val_acc=0
total=0
for features,labels in dataloader:
inputs, targets = features.to(device), labels.to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 total+=len(targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 _, predicted = torch.max(outputs, 1)
 val_acc+=(( predicted == targets ).sum().item())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 val_acc=(val_acc/total)*100
return final_loss, valid_preds,val_acc 
<define_variables>
learn.fit_one_cycle(4,1e-4 )
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 12
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-8
seed=42 
<choose_model_class>
learn.unfreeze()
learn.lr_find()
learn.recorder.plot()
def run_training(seed):
train_loader, valid_loader= create_dataloaders(seed=seed)
 model=Model()
model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE,weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e2,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(train_loader))
loss_fn = nn.CrossEntropyLoss()
for epoch in range(EPOCHS):
train_loss,train_acc = train_fn(model, optimizer,scheduler, loss_fn, train_loader, DEVICE)
 print(f"EPOCH: {epoch}, train_loss: {train_loss},, train_accuracy:{train_acc}")
 val_loss, val_preds, val_acc = valid_fn(model, loss_fn, valid_loader, DEVICE)
 print(f"EPOCH: {epoch}, valid_loss: {val_loss}, val_accuracy:{val_acc}")
 test_pred = torch.LongTensor()
testdataset = TestDataset(test_data)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 for features in testloader:
features=features.to(DEVICE)
 outputs=model(features)
 _, predicted = torch.max(outputs, 1)
 test_pred = torch.cat(( test_pred.to(DEVICE), predicted.to(DEVICE)) , dim=0)
 pred_df['predict'] = test_pred.cpu().numpy()<train_model>
learn.fit_one_cycle(2,max_lr = slice(1e-5/3,1e-5))
pred_df = sample_data.copy()
run_training(seed)
<data_type_conversions>
learn.save('stage-3' )
final_pred = pred_df['predict']
sample_data.Label = final_pred.astype(int)
 sample_data.head()<save_to_csv>
preds,y = learn.TTA()
acc = accuracy(preds, y)
 print('The validation accuracy is {} %.'.format(acc * 100))
pred_score = auc_score(preds,y ).item()
print('The validation AUC is {}.'.format(pred_score))
sample_data.to_csv('./submission.csv', index=False )<load_from_csv>
interp = ClassificationInterpretation.from_learner(learn )
data_train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 X_test = pd.read_csv(".. /input/digit-recognizer/test.csv" )<prepare_x_and_y>
preds,y = learn.TTA()
acc = accuracy(preds, y)
 print('The validation accuracy is {} %.'.format(acc * 100))
pred_score = auc_score(preds,y ).item()
print('The validation AUC is {}.'.format(pred_score))
X, y = data_train.drop(labels = ["label"],axis = 1)/255.,data_train["label"]
X_test = X_test/255 .<choose_model_class>
def generateSubmission(learner):
submissions = pd.read_csv('.. /input/sample_submission.csv')
 id_list = list(submissions.id)
 preds,y = learner.TTA(ds_type=DatasetType.Test)
 pred_list = list(preds[:,1])
 pred_dict = dict(( key, value.item())for(key, value)in zip(learner.data.test_ds.items,pred_list))
pred_ordered = [pred_dict[Path('.. /input/test/' + id + '.tif')] for id in id_list]
submissions = pd.DataFrame({'id':id_list,'label':pred_ordered})
 submissions.to_csv("submission_transferLearning_{}.csv".format(pred_score),index = False )