Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<categorify>
warnings.filterwarnings("ignore")
 %load_ext autoreload
%autoreload
cat_feat_to_encode = train.columns.tolist()
smoothing=50.0
oof = pd.DataFrame([])
 for tr_idx, oof_idx in StratifiedKFold(
 n_splits=5, random_state=1, shuffle=True ).split(
 train, train_y):
ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train.iloc[tr_idx, :], train_y.iloc[tr_idx])
 oof = oof.append(ce_target_encoder.transform(train.iloc[oof_idx, :]), ignore_index=False)
 ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(train, train_y)
 train = oof.sort_index()
test = ce_target_encoder.transform(test )<train_model>
model_path='.'
path='.. /input/'
train_folder=f'{path}train'
test_folder=f'{path}test'
train_lbl=f'{path}train_labels.csv'
ORG_SIZE=96
bs=64
num_workers=None
sz=96
clf = lgb.train(
 params={
'max_depth': 2,
'num_leaves': 150,
'reg_alpha': 0.6,
'reg_lambda': 0.6,
'objective': 'binary',
"boosting_type": "gbdt",
"metric": 'auc',
"verbosity": -1,
'random_state': 1},
train_set=lgb.Dataset(train, label=train_y),
num_boost_round=700 )<save_to_csv>
df_trn=pd.read_csv(train_lbl )
pd.DataFrame({'id': test_id, 'target': clf.predict(test)} ).to_csv(
 'sub_' + str(datetime.now().strftime('%Y-%m-%d_%H-%M-%S')) + '.csv',
index=False )<set_options>
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=.0, max_zoom=.1,
max_lighting=0.05, max_warp=0.)
warnings.filterwarnings("ignore")
<load_from_csv>
data = ImageDataBunch.from_csv(path,csv_labels=train_lbl,folder='train', ds_tfms=tfms, size=sz, suffix='.tif',test=test_folder,bs=bs);
stats=data.batch_stats()
data.normalize(stats )
train = pd.read_csv('.. /input/cat-in-the-dat/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat/test.csv')
 print(train.shape)
 print(test.shape )<feature_engineering>
data.show_batch(rows=5, figsize=(12,9))
Q1 = train.quantile(0.25)
 Q3 = train.quantile(0.75)
 IQR = Q3 - Q1
train = train[~(( train <(Q1 - 1.5 * IQR)) |(train >(Q3 + 1.5 * IQR)) ).any(axis=1)]<drop_column>
from sklearn.metrics import roc_auc_score
target = train['target']
train_id = train['id']
test_id = test['id']
train.drop(['target', 'id'], axis=1, inplace=True)
 test.drop('id', axis=1, inplace=True)
 print(train.shape)
 print(test.shape )<prepare_output>
def auc_score(y_pred,y_true,tens=True):
score=roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])
 if tens:
score=tensor(score)
 else:
score=score
return score
def description(df):
summary = pd.DataFrame(df.dtypes,columns=['dtypes'])
 summary = summary.reset_index()
summary['Name'] = summary['index']
summary = summary[['Name','dtypes']]
summary['Missing'] = df.isnull().sum().values
summary['Uniques'] = df.nunique().values
summary['First Value'] = df.iloc[0].values
summary['Second Value'] = df.iloc[1].values
summary['Third Value'] = df.iloc[2].values
return summary
description(train )<categorify>
from torchvision.models import *
bin_dict = {'T':1, 'F':0, 'Y':1, 'N':0}
train['bin_3'] = train['bin_3'].map(bin_dict)
 train['bin_4'] = train['bin_4'].map(bin_dict)
 test['bin_3'] = test['bin_3'].map(bin_dict)
 test['bin_4'] = test['bin_4'].map(bin_dict )<categorify>
learn = create_cnn(
 data,
densenet201,
path='.',
metrics=[auc_score],
ps=0.5
)
dummies = pd.concat([train, test], axis=0, sort=False)
 print(f'Shape before dummy transformation: {dummies.shape}')
 dummies = pd.get_dummies(dummies, columns=['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4'],\
prefix=['nom_0', 'nom_1', 'nom_2', 'nom_3', 'nom_4'], drop_first=True)
 print(f'Shape after dummy transformation: {dummies.shape}')
 train, test = dummies.iloc[:train.shape[0], :], dummies.iloc[train.shape[0]:, :]
del dummies
print(train.shape)
 print(test.shape )<categorify>
lr = 1e-04
def date_cyc_enc(df, col, max_vals):
df[col + '_sin'] = np.sin(2 * np.pi * df[col]/max_vals)
 df[col + '_cos'] = np.cos(2 * np.pi * df[col]/max_vals)
 return df
train = date_cyc_enc(train, 'day', 7)
 test = date_cyc_enc(test, 'day', 7)
 train = date_cyc_enc(train, 'month', 12)
 test = date_cyc_enc(test, 'month', 12)
 print(train.shape)
 print(test.shape )<feature_engineering>
learn.fit_one_cycle(1,lr)
 learn.recorder.plot()
learn.recorder.plot_losses()
train['ord_5_oe_add'] = train['ord_5'].apply(lambda x:sum([(string.ascii_letters.find(letter)+1)for letter in x]))
test['ord_5_oe_add'] = test['ord_5'].apply(lambda x:sum([(string.ascii_letters.find(letter)+1)for letter in x]))
train['ord_5_oe_join'] = train['ord_5'].apply(lambda x:float(''.join(str(string.ascii_letters.find(letter)+1)for letter in x)))
 test['ord_5_oe_join'] = test['ord_5'].apply(lambda x:float(''.join(str(string.ascii_letters.find(letter)+1)for letter in x)))
 train['ord_5_oe1'] = train['ord_5'].apply(lambda x:(string.ascii_letters.find(x[0])+1))
test['ord_5_oe1'] = test['ord_5'].apply(lambda x:(string.ascii_letters.find(x[0])+1))
train['ord_5_oe2'] = train['ord_5'].apply(lambda x:(string.ascii_letters.find(x[1])+1))
test['ord_5_oe2'] = test['ord_5'].apply(lambda x:(string.ascii_letters.find(x[1])+1))
for col in ['ord_5_oe1', 'ord_5_oe2', 'ord_5_oe_add', 'ord_5_oe_join']:
train[col]= train[col].astype('float64')
 test[col]= test[col].astype('float64')
 print(train.shape)
 print(test.shape )<categorify>
learn.unfreeze()
learn.lr_find()
high_card_feats = ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']
for col in high_card_feats:
train[f'hash_{col}'] = train[col].apply(lambda x: hash(str(x)) % 5000)
 test[f'hash_{col}'] = test[col].apply(lambda x: hash(str(x)) % 5000)
 if train[col].dtype == 'object' or test[col].dtype == 'object':
lbl = LabelEncoder()
lbl.fit(list(train[col].values)+ list(test[col].values))
train[f'le_{col}'] = lbl.transform(list(train[col].values))
test[f'le_{col}'] = lbl.transform(list(test[col].values))
print(train.shape)
 print(test.shape )<drop_column>
learn.fit_one_cycle(1,slice(1e-4,1e-3))
col = high_card_feats + ['ord_5','day', 'month']
train.drop(col, axis=1, inplace=True)
 test.drop(col, axis=1, inplace=True)
 print(train.shape)
 print(test.shape )<categorify>
m = learn.model.eval()
def get_optimized_column(column):
if not np.issubdtype(column.dtypes, np.number):
return column
integers = [np.int8, np.int16, np.int32, np.int64]
floats = [np.float16, np.float32, np.float64]
max = column.max()
relevant_types = integers if np.issubdtype(column.dtypes, np.integer)else floats
for dtype in relevant_types:
if dtype(max)== max:
return column.astype(dtype)
 return column
def reduce_mem_usage(df, verbose=True):
start_mem = df.memory_usage().sum() / 1024**2
for column in df.columns:
df[column] = get_optimized_column(df[column])
 end_mem = df.memory_usage().sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df
train = reduce_mem_usage(train)
 test = reduce_mem_usage(test)
 description(train )<train_model>
idx = 4
def runCatBoost(train_pool, validate_pool, params):
print('Train CatBoost model')
 model = CatBoostClassifier(**params)
 model.fit(train_pool, eval_set=validate_pool, plot=True)
 return model
def predict(model, pool):
print('Predict')
 pred_test_y = model.predict_proba(X_test)[:, 1]
return pred_test_y
X_train, X_val, y_train, y_val = train_test_split(
 train, target,
test_size = 0.3,
random_state = 42 
)
 cat_features = ['bin_0','bin_1','bin_2','bin_3','bin_4','ord_0','ord_1','ord_2','ord_3','ord_4']
train_pool = Pool(train, target, cat_features = cat_features)
 validate_pool = Pool(X_val, y_val, cat_features = cat_features)
 test_pool = Pool(data=test, cat_features = cat_features )<choose_model_class>
from fastai.callbacks.hooks import *
params = {
'iterations': 15000,
'learning_rate': 0.1,
'eval_metric': 'AUC',
'random_seed': 42,
'task_type':'GPU',
'l2_leaf_reg': 10,
'logging_level': 'Silent',
'use_best_model': True,
'custom_loss': ['AUC'],
'metric_period': 50,
'bagging_temperature' : 0.2,
'l2_leaf_reg': 10
}
model1 = runCatBoost(train_pool, validate_pool, params )<train_model>
def hooked_backward(cat=y):
with hook_output(m[0])as hook_a:
with hook_output(m[0], grad = True)as hook_g:
preds = m(xb)
 preds[0,int(cat)].backward()
return hook_a, hook_g
params.update({
'random_seed': 42,
'learning_rate': 0.05,
'l2_leaf_reg': 5,
'depth': 4
})
<train_model>
preds,y=learn.get_preds()
pred_score=auc_score(preds,y)
 pred_score
params.update({
'random_seed': 44,
'learning_rate': 0.025,
'l2_leaf_reg': 3,
'depth': 7
})
<train_model>
preds,y=learn.TTA()
pred_score_tta=auc_score(preds,y)
 pred_score_tta
params.update({
'iterations': 15000,
'random_seed': 45,
'learning_rate': 0.0125,
'l2_leaf_reg': 10,
'depth': 5
})
<save_to_csv>
preds_test,y_test=learn.get_preds(ds_type=DatasetType.Test )
def submit(model, pool, i):
print('Predict ' + str(i))
results = model.predict_proba(pool)[:, 1]
submission = pd.DataFrame({'id': test_id, 'target': results})
 submission.to_csv('submission' + str(i)+ '.csv', index=False)
 submit(model1, test_pool,1)
<import_modules>
preds_test_tta,y_test_tta=learn.TTA(ds_type=DatasetType.Test )
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score,classification_report
import seaborn as sns
import pandas as pd
import numpy as np 
<load_from_csv>
sub=pd.read_csv(f'{path}/sample_submission.csv' ).set_index('id')
 sub.head()
submission = pd.read_csv("/kaggle/input/cat-in-the-dat/sample_submission.csv")
 train = pd.read_csv("/kaggle/input/cat-in-the-dat/train.csv")
 test = pd.read_csv("/kaggle/input/cat-in-the-dat/test.csv")
 labels = train['target'].values
train_id = train.pop("id")
 test_id = test.pop("id" )<concatenate>
clean_fname=np.vectorize(lambda fname: str(fname ).split('/')[-1].split('.')[0])
 fname_cleaned=clean_fname(data.test_ds.items)
 fname_cleaned=fname_cleaned.astype(str )
data = pd.concat([train.drop('target',axis=1), test])
 totaal=train.append(test )<categorify>
sub.loc[fname_cleaned,'label']=to_np(preds_test[:,1])
 sub.to_csv(f'submission_{pred_score}.csv' )
cols=[ci for ci in train.columns if ci not in ['id','index','target']]
coltype=train.dtypes
for ci in cols:
if(coltype[ci]=="object"):
codes=totaal[[ci,'target']].groupby(ci ).mean().sort_values("target")
 codesdict=codes.target.to_dict()
totaal[ci]=totaal[ci].map(codesdict)
<feature_engineering>
sub.loc[fname_cleaned,'label']=to_np(preds_test_tta[:,1])
 sub.to_csv(f'submission_{pred_score_tta}.csv' )