Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<data_type_conversions>
data = pd.read_csv('/kaggle/input/train_labels.csv')
 train_path = '/kaggle/input/train/'
test_path = '/kaggle/input/test/'
data['label'].value_counts()
X = X.fillna(-999)
 X_test = X_test.fillna(-999 )<categorify>
def readImage(path):
bgr_img = cv2.imread(path)
 b,g,r = cv2.split(bgr_img)
 rgb_img = cv2.merge([r,g,b])
 return rgb_img
for col in X.columns:
if X[col].dtype=='object' or X_test[col].dtype=='object':
lbl = LabelEncoder()
lbl.fit(list(X[col].values)+ list(X_test[col].values))
X[col] = lbl.transform(list(X[col].values))
X_test[col] = lbl.transform(list(X_test[col].values))<drop_column>
ORIGINAL_SIZE = 96
CROP_SIZE = 90
RANDOM_ROTATION = 3
RANDOM_SHIFT = 2
RANDOM_BRIGHTNESS = 7
RANDOM_CONTRAST = 5
RANDOM_90_DEG_TURN = 1
def readCroppedImage(path, augmentations = True):
bgr_img = cv2.imread(path)
 b,g,r = cv2.split(bgr_img)
 rgb_img = cv2.merge([r,g,b])
 if(not augmentations):
return rgb_img / 255
rotation = random.randint(-RANDOM_ROTATION,RANDOM_ROTATION)
 if(RANDOM_90_DEG_TURN == 1):
rotation += random.randint(-1,1)* 90
M = cv2.getRotationMatrix2D(( 48,48),rotation,1)
 rgb_img = cv2.warpAffine(rgb_img,M,(96,96))
x = random.randint(-RANDOM_SHIFT, RANDOM_SHIFT)
 y = random.randint(-RANDOM_SHIFT, RANDOM_SHIFT)
 start_crop =(ORIGINAL_SIZE - CROP_SIZE)// 2
end_crop = start_crop + CROP_SIZE
rgb_img = rgb_img[(start_crop + x):(end_crop + x),(start_crop + y):(end_crop + y)] / 255
flip_hor = bool(random.getrandbits(1))
flip_ver = bool(random.getrandbits(1))
if(flip_hor):
rgb_img = rgb_img[:, ::-1]
if(flip_ver):
rgb_img = rgb_img[::-1, :]
br = random.randint(-RANDOM_BRIGHTNESS, RANDOM_BRIGHTNESS)/ 100.
 rgb_img = rgb_img + br
cr = 1.0 + random.randint(-RANDOM_CONTRAST, RANDOM_CONTRAST)/ 100.
 rgb_img = rgb_img * cr
rgb_img = np.clip(rgb_img, 0, 1.0)
 return rgb_img
%%time
X = reduce_mem_usage(X)
 X_test = reduce_mem_usage(X_test )<init_hyperparams>
train_df = data.set_index('id')
 train_names = train_df.index.values
train_labels = np.asarray(train_df['label'].values)
 tr_n, tr_idx, val_n, val_idx = train_test_split(train_names, range(len(train_names)) , test_size=0.1, stratify=train_labels, random_state=123 )
params = {'bagging_fraction': 0.8831783305477362,
'colsample_bytree': 0.5941836821402656,
'feature_fraction': 0.8359307480774029,
'learning_rate': 0.23836691517558517,
'max_depth': 13,
'min_child_samples': 105.54076348952397,
'min_child_weight': 0.005812630846994812,
'min_data_in_leaf': 24,
'num_leaves': 496,
'reg_alpha': 1.229972928485176,
'reg_lambda': 1.3080809161627962,
'subsample': 0.4071603440308888,
'metric':'auc',
'bagging_seed': 11,
'n_estimators': 5000,
}<find_best_model_class>
arch = densenet169
BATCH_SIZE = 128
sz = CROP_SIZE
MODEL_PATH = str(arch ).split() [1]
%%time
NFOLDS = 7
folds = KFold(n_splits=NFOLDS)
 columns = X.columns
splits = folds.split(X, y)
 y_preds = np.zeros(X_test.shape[0])
 y_oof = np.zeros(X.shape[0])
 score = 0
for fold_n,(train_index, valid_index)in enumerate(splits):
clf = lgb.LGBMClassifier(**params)
 X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
clf.fit(X_train, y_train)
 y_pred_valid = clf.predict_proba(X_valid)[:,1]
y_oof[valid_index] = y_pred_valid
print(f"Fold {fold_n + 1} | AUC: {roc_auc_score(y_valid, y_pred_valid)}")
 score += roc_auc_score(y_valid, y_pred_valid)/ NFOLDS
y_preds += clf.predict_proba(X_test)[:,1] / NFOLDS
del X_train, X_valid, y_train, y_valid
gc.collect()
print(f"
Mean AUC = {score}")
 print(f"Out of folds AUC = {roc_auc_score(y, y_oof)}" )<prepare_output>
train_dict = {'name': train_path + train_names, 'label': train_labels}
df = pd.DataFrame(data=train_dict)
 test_names = []
for f in os.listdir(test_path):
test_names.append(test_path + f)
 df_test = pd.DataFrame(np.asarray(test_names), columns=['name'] )
preds = y_preds<save_to_csv>
class MyImageItemList(ImageList):
def open(self, fn:PathOrStr)->Image:
img = readCroppedImage(fn.replace('/./','' ).replace('//','/'))
return vision.Image(px=pil2tensor(img, np.float32))
sub['isFraud'] = preds
sub.to_csv('v3_noblend.csv', index=False )<save_to_csv>
imgDataBunch =(MyImageItemList.from_df(path='/', df=df, suffix='.tif')
.split_by_idx(val_idx)
.label_from_df(cols='label')
.add_test(MyImageItemList.from_df(path='/', df=df_test)) 
.transform(tfms=[[],[]], size=sz)
.databunch(bs=BATCH_SIZE)
.normalize([tensor([0.702447, 0.546243, 0.696453]), tensor([0.238893, 0.282094, 0.216251])])
 )
safebox = pd.read_csv('.. /input/safe-box/blend01.csv')
 safebox_preds = safebox['isFraud'].values
sub['isFraud'] =(preds + 2*safebox_preds)/3
blend01 =(preds + 2*safebox_preds)/3
sub.to_csv('v3_safebox_blend01.csv', index=False )<save_to_csv>
def getLearner() :
return create_cnn(imgDataBunch, arch, pretrained=True, path='.', metrics=accuracy, ps=0.5, callback_fns=ShowGraph)
 learner = getLearner()
safebox = pd.read_csv('.. /input/safe-box/blend02.csv')
 safebox_preds = safebox['isFraud'].values
sub['isFraud'] =(preds + 2*safebox_preds)/3
blend02 =(preds + 2*safebox_preds)/3
sub.to_csv('v3_safebox_blend02.csv', index=False )<save_to_csv>
lrs = []
losses = []
wds = []
iter_count = 600
learner.lr_find(wd=1e-6, num_it=iter_count)
 lrs.append(learner.recorder.lrs)
 losses.append(learner.recorder.losses)
 wds.append('1e-6')
 learner = getLearner()
learner.lr_find(wd=1e-4, num_it=iter_count)
 lrs.append(learner.recorder.lrs)
 losses.append(learner.recorder.losses)
 wds.append('1e-4')
 learner = getLearner()
learner.lr_find(wd=1e-2, num_it=iter_count)
 lrs.append(learner.recorder.lrs)
 losses.append(learner.recorder.losses)
 wds.append('1e-2')
 learner = getLearner()
safebox = pd.read_csv('.. /input/safe-box/submission.csv')
 safebox_preds = safebox['isFraud'].values
sub['isFraud'] =(preds + 2*safebox_preds)/3
sub.to_csv('v3_safebox_sub.csv', index=False )<save_to_csv>
max_lr = 2e-2
wd = 1e-4
learner.fit_one_cycle(cyc_len=8, max_lr=max_lr, wd=wd )
sub['isFraud'] =(blend01 + blend02)/2
sub.to_csv('v3_safebox_b1_and_b2.csv', index=False )<save_to_csv>
learner.save(MODEL_PATH + '_stage1' )
safebox = pd.read_csv('.. /input/safe-box/blend02.csv')
 safebox_preds = safebox['isFraud'].values
sub['isFraud'] =(preds + safebox_preds)/2
sub.to_csv('v3_safebox_equal_parts.csv', index=False )<save_to_csv>
learner.fit_one_cycle(cyc_len=12, max_lr=slice(4e-5,4e-4))
safebox = pd.read_csv('.. /input/stackers-blend-top-4/All_Blends.csv')
 safebox_preds = safebox['isFraud'].values
sub['isFraud'] =(preds + safebox_preds)/2
sub.to_csv('v3_stacking_equal_parts.csv', index=False )<set_options>
learner.save(MODEL_PATH + '_stage2' )
%reload_ext autoreload
%autoreload 2
%matplotlib inline<import_modules>
from fastai.vision import *
from fastai.metrics import error_rate<load_pretrained>
preds,y, loss = learner.get_preds(with_loss=True)
 acc = accuracy(preds, y)
 print('The accuracy is {0} %.'.format(acc))
DATA_PATH = ".. /input/dogs-vs-cats-redux-kernels-edition"
TRAIN_PATH = DATA_PATH + "/train"
TEST_PATH = DATA_PATH + "/test"
path = untar_data("",DATA_PATH,DATA_PATH); path; path.ls()<define_variables>
def hooked_backward(m, oneBatch, cat):
with hook_output(m[0])as hook_a:
with hook_output(m[0], grad=True)as hook_g:
preds = m(oneBatch)
 preds[0,int(cat)].backward()
return hook_a,hook_g
fnames = get_image_files(TRAIN_PATH)
 pat = r'([^/]+)\.\d+\.jpg$'
fnames[:5]<load_pretrained>
probs = np.exp(preds[:,1])
 fpr, tpr, thresholds = roc_curve(y, probs, pos_label=1)
 roc_auc = auc(fpr, tpr)
 print('ROC area is {0}'.format(roc_auc))
bs=64
data = ImageDataBunch.from_name_re(TRAIN_PATH, fnames, pat, ds_tfms=get_transforms() , size=499, bs=bs
).normalize(imagenet_stats)
 data.show_batch(rows=3, figsize=(7,6))<choose_model_class>
learner.load(MODEL_PATH + '_stage2')
 n_aug = 12
preds_n_avg = np.zeros(( len(learner.data.test_ds.items),2))
for n in tqdm_notebook(range(n_aug), 'Running TTA...'):
preds,y = learner.get_preds(ds_type=DatasetType.Test, with_loss=False)
 preds_n_avg = np.sum([preds_n_avg, preds.numpy() ], axis=0)
 preds_n_avg = preds_n_avg / n_aug
learn = cnn_learner(data, models.resnet18, metrics=error_rate, pretrained=True )<define_variables>
print('Negative and Tumor Probabilities: ' + str(preds_n_avg[0]))
tumor_preds = preds_n_avg[:, 1]
print('Tumor probability: ' + str(tumor_preds[0]))
class_preds = np.argmax(preds_n_avg, axis=1)
 classes = ['Negative','Tumor']
print('Class prediction: ' + classes[class_preds[0]] )
learn.model_dir = "/tmp"
os.access('/tmp', os.W_OK )<train_model>
SAMPLE_SUB = '/kaggle/input/sample_submission.csv'
sample_df = pd.read_csv(SAMPLE_SUB)
 sample_list = list(sample_df.id)
 pred_list = [p for p in tumor_preds]
pred_dic = dict(( key, value)for(key, value)in zip(learner.data.test_ds.items, pred_list))
pred_list_cor = [pred_dic['///kaggle/input/test/' + id + '.tif'] for id in sample_list]
df_sub = pd.DataFrame({'id':sample_list,'label':pred_list_cor})
 df_sub.to_csv('{0}_submission.csv'.format(MODEL_PATH), header=True, index=False )
learn.fit_one_cycle(3)
 learn.save('stage-1' )<find_best_params>
imgDataBunch.export(fname='./export.pkl' )
<train_model><EOS> 
sz = 68
def imageToTensorImage(path):
bgr_img = cv2.imread(path)
 b,g,r = cv2.split(bgr_img)
 rgb_img = cv2.merge([r,g,b])
 H,W,C = rgb_img.shape
rgb_img = rgb_img[(H-sz)//2:(sz +(H-sz)//2),(H-sz)//2:(sz +(H-sz)//2),:] / 256
return vision.Image(px=pil2tensor(rgb_img, np.float32))
img = imageToTensorImage('/kaggle/input/test/0eb051700fb6b1bf96188f36c8e4889598c6a157.tif')
 img.show(y=learner.predict(img)[0])
 classes = ['negative', 'tumor']
print('This is a ' + classes[int(learner.predict(img)[0])] + ' tissue image.' )