Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<compute_test_metric>
import numpy as np
import pandas as pd
from pathlib import Path
from fastai import *
from fastai.vision import *
import torchvision
import torch
print("Overall AUC={}".format(metrics.roc_auc_score(train.target.values, oof_preds)) )<save_to_csv>
data_root_path = Path(".. /input" )
test_preds /= 5
test_ids = test.id.values
print("Saving submission file")
 submission = pd.DataFrame.from_dict({
'id': test_ids,
'target': test_preds
})
 submission.to_csv("submission.csv", index=False )<import_modules>
train_df = pd.read_csv(data_root_path/"train_labels.csv")
 test_df = pd.read_csv(data_root_path/"sample_submission.csv")
 train_df.id = train_df.id + '.tif'
test_df.id = test_df.id + '.tif'
import pandas as pd
import numpy as np
import category_encoders as ce
import lightgbm as lgb
from sklearn import linear_model
from sklearn.model_selection import StratifiedKFold
import gc<set_options>
transforms = get_transforms(
 do_flip=True,
flip_vert=True,
max_rotate=15.0,
max_lighting=0.2,
max_warp=0.2 
)
 train_imgs = ImageList.from_df(train_df, path=data_root_path, folder='train')
 test_imgs = ImageList.from_df(test_df, path=data_root_path, folder='test')
 train_imgs =(train_imgs 
.split_by_rand_pct(0.01)
.label_from_df() 
.add_test(test_imgs)
.transform(transforms, size=128)
.databunch(path='.', bs=64, device= torch.device('cuda:0')) 
.normalize(imagenet_stats))
warnings.filterwarnings("ignore", category=DeprecationWarning)
 warnings.filterwarnings("ignore", category=UserWarning)
 warnings.filterwarnings("ignore", category=FutureWarning)
<load_from_csv>
learn = cnn_learner(train_imgs, torchvision.models.densenet169, metrics=[error_rate, accuracy] )
train = pd.read_csv('.. /input/cat-in-the-dat/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat/test.csv')
 print(train.target.value_counts() [0]/300000, train.target.value_counts() [1]/300000,)
 train.sort_index(inplace=True)
 train_y = train['target']
test_id = test['id']
train.drop(['target', 'id'], axis=1, inplace=True)
 test.drop('id', axis=1, inplace=True )<set_options>
lr = 3e-02
learn.fit_one_cycle(10, slice(lr))
HTML('<iframe width="680" height="620" src="https://www.youtube.com/embed/8odLEbSGXoI" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>')
<categorify>
preds,_ = learn.get_preds(ds_type=DatasetType.Test )
cat_feat_to_encode = train.columns.tolist()
smoothing=0.50
oof = pd.DataFrame([])
 for tr_idx, oof_idx in StratifiedKFold(n_splits=5, random_state=1, shuffle=True ).split(train, train_y):
ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train.iloc[tr_idx, :], train_y.iloc[tr_idx])
 oof = oof.append(ce_target_encoder.transform(train.iloc[oof_idx, :]), ignore_index=False)
 ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train, train_y)
 train = oof.sort_index()
test = ce_target_encoder.transform(test )<train_model>
test_df.label = preds.numpy() [:, 0]
glm = linear_model.LogisticRegression(
 random_state=1, solver='lbfgs', max_iter=2019, fit_intercept=True,
penalty='none', verbose=0)
 glm.fit(train, train_y )<save_to_csv>
test_df['id'] = test_df['id'].str.replace('.tif','' )
<load_from_csv><EOS> 
test_df.to_csv('submission.csv', index=False )