Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<train_model>
!pip install fastai==0.7.0 --no-deps
!pip install torch==0.4.1 torchvision==0.2.1
learn.unfreeze()
learn.fit_one_cycle(2, max_lr=slice(1e-6,1e-5))<save_model>
from fastai.conv_learner import *
from fastai.dataset import *
import pandas as pd
import numpy as np
import os
from sklearn.model_selection import train_test_split
learn.save('stage-tunned-34' )<define_variables>
MODEL_PATH = 'Dn121_v1'
TRAIN = '.. /input/train/'
TEST = '.. /input/test/'
LABELS = '.. /input/train_labels.csv'
SAMPLE_SUB = '.. /input/sample_submission.csv'
ORG_SIZE=96
BATCH_SIZE = 128
test_fnames = get_image_files(TEST_PATH)
 test_fnames[:5]<predict_on_test>
arch = dn121
nw = 4
img = open_image(test_fnames[3])
 learn.predict(img )<predict_on_test>
train_df = pd.read_csv(LABELS ).set_index('id')
 train_names = train_df.index.values
train_labels = np.asarray(train_df['label'].values)
 print("Number of positive samples = {:.4f}%".format(np.count_nonzero(train_labels)*100/len(train_labels)))
 test_names = [f.replace(".tif","")for f in os.listdir(TEST)]
tr_n, val_n = train_test_split(train_names, test_size=0.15, random_state=42069)
 print(len(tr_n), len(val_n))
learn.predict(img)[2][1].item()<predict_on_test>
class HCDDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.train_df = train_df
super().__init__(fnames, transform, path)
 def get_x(self, i):
img = open_image(os.path.join(self.path, self.fnames[i]+".tif"))
img = img[(ORG_SIZE-self.sz)//2:(ORG_SIZE+self.sz)//2,(ORG_SIZE-self.sz)//2:(ORG_SIZE+self.sz)//2,:]
return img
def get_y(self, i):
if(self.path == TEST): return 0
return self.train_df.loc[self.fnames[i]]['label']
def get_c(self):
return 2
submission = {"id":[],"label":[]}
idpat = r''
for f in test_fnames:
img = open_image(f)
 fname = f.name
id = fname[:fname.rfind(".")]
submission["id"].append(id)
 submission["label"].append(learn.predict(img)[2][1].item() )<save_to_csv>
def get_data(sz, bs):
aug_tfms = [RandomRotate(20, tfm_y=TfmType.NO),
RandomDihedral(tfm_y=TfmType.NO)]
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.NO,
aug_tfms=aug_tfms)
 ds = ImageData.get_ds(HCDDataset,(tr_n[:-(len(tr_n)% bs)], TRAIN), 
(val_n, TRAIN), tfms, test=(test_names, TEST))
md = ImageData("./", ds, bs, num_workers=nw, classes=None)
 return md
result = pd.DataFrame(submission)
 result.to_csv('submission.csv', index=False )<load_from_csv>
md = get_data(96, BATCH_SIZE)
 learn = ConvLearner.pretrained(arch, md)
 learn.opt_fn = optim.Adam
train = pd.read_csv('/kaggle/input/cat-in-the-dat/train.csv')
 test = pd.read_csv('/kaggle/input/cat-in-the-dat/test.csv' )<count_unique_values>
learn.fit(lr, 1, cycle_len=2)
 learn.unfreeze()
lrs = np.array([1e-4, 5e-4, 1.2e-3])
 learn.fit(lrs, 1, cycle_len=5, use_clr=(20, 16))
learn.fit(lrs/4, 1, cycle_len=5, use_clr=(10, 8))
learn.fit(lrs/16, 1, cycle_len=5, use_clr=(10, 8))
train.nunique()<count_missing_values>
preds_t,y_t = learn.TTA(is_test=True, n_aug=8)
 preds_t = np.stack(preds_t, axis=-1)
 preds_t = np.exp(preds_t)
 preds_t = preds_t.mean(axis=-1)[:,1]
train.isna().sum()<define_variables>
sample_df = pd.read_csv(SAMPLE_SUB)
 sample_list = list(sample_df.id)
 pred_list = [p for p in preds_t]
pred_dic = dict(( key, value)for(key, value)in zip(learn.data.test_ds.fnames,pred_list))
pred_list_cor = [pred_dic[id] for id in sample_list]
df = pd.DataFrame({'id':sample_list,'label':pred_list_cor})
 df.to_csv('submission.csv'.format(MODEL_PATH), header=True, index=False)