Kaggle competition: Aerial Cactus Identification
<SOS> metric: AUC Kaggle data source: aerial-cactus-identification<define_variables>
import numpy as np
import pandas as pd
def binning(sort):
r = dict()
for i, vals in enumerate(sort):
p = {v: i for v in vals}
r = {**r, **p}
return r
poor = [
'MeadowV', 'IDOTRR', 'BrDale',
]
middle_low = [
'OldTown', 'Edwards','BrkSide',
'Sawyer', 'Blueste', 'SWISU',
'NAmes',
]
middle_med = [
'NPkVill', 'Mitchel',
]
middle_hig = [
'SawyerW', 'Gilbert', 'NWAmes',
'Blmngtn', 'CollgCr', 'ClearCr',
'Crawfor',
'Veenker','Somerst','Timber',
]
high = [
'StoneBr', 'NoRidge', 'NridgHt',
]
sort = [poor, middle_low, middle_med, middle_hig, high]
neig_bin = binning(sort)
 rank_dict = {
'Neighborhood': neig_bin,
}<define_variables>
from pathlib import Path
from fastai import *
from fastai.vision import *
import torch
sort = [
['180', '30', '45'],
['190', '50', '90', '85', '40'],
['160', '70', '20', '75', '80', '150'],
['120', '60'],
]
rank_dict['MSSubClass'] = binning(sort )<define_variables>
data_folder = Path(".. /input" )
sort = [
['2fmCon'],
['Duplex', 'Twnhs' ],
['1Fam','TwnhsE'],
]
rank_dict['BldgType'] = binning(sort )<define_variables>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/sample_submission.csv" )
sort = [
['AdjLand'],
['Abnorml', 'Family'],
['Alloca','Normal'],
[],
['Partial']
]
rank_dict['SaleCondition'] = binning(sort)
 rank_dict['SaleCondition'] = {'AdjLand': 0,
'Abnorml': 1, 'Family': 2,
'Alloca': 3, 'Normal': 4,
'Partial': 5}<feature_engineering>
train_image_list_bunch.show_batch(rows=4, figsize=(7,6))
def make_rank(df, ranks):
for col, d in ranks.items() :
df[col] = df[col].apply(lambda x: d.get(x, x))
return df
dataset = apply(dataset, make_rank, rank_dict )<categorify>
learn = cnn_learner(train_image_list_bunch, models.densenet161, metrics=[error_rate, accuracy] )
mistakes = {
'Brk Cmn': 'BrkComm',
'CmentBd': 'CemntBd',
}
def correct_mistakes(df, cols, mistake_dict):
for col in cols:
for mistake, correct in mistake_dict.items() :
df[col] = df[col].replace(mistake, correct)
 return df
dataset = apply(dataset, correct_mistakes, ecols, mistakes )<feature_engineering>
learn.lr_find()
learn.recorder.plot()
def make_exterior(df, values):
E1, E2 = ecols = ['Exterior1st', 'Exterior2nd']
for v in values:
df['Exterior_' + v] =(df[E1] == v)|(df[E2] == v)
 df = df.drop(ecols, axis=1, inplace=True)
 return df
values = set(X[E1].unique()
).union(set(X[E2].unique()))
dataset = apply(dataset, make_exterior, values )<drop_column>
lr = 1e-01
learn.fit_one_cycle(5, slice(lr))
noise_cols = ['LotConfig', 'RoofStyle', 'RoofMatl',
'Heating', 'Electrical', 'SaleType',
]
dataset = apply(dataset, make_drop, noise_cols)
<categorify>
preds,_ = learn.get_preds(ds_type=DatasetType.Test )
X = pd.get_dummies(X)
 test = pd.get_dummies(test)
 print(list(X.columns))<train_model>
test_df.has_cactus = preds.numpy() [:, 0]
row = X.shape[0]
overfit_min = row - math.ceil(row * 5 / 10000)
 c =(X == 0 ).sum() >= overfit_min
overfit_cols = set(c[c].index)
 X.drop(overfit_cols, axis=1, inplace=True)
 both = set(test.columns ).intersection(overfit_cols)
 test.drop(both, axis=1, inplace=True)
 test_overfit = list(set(X.columns)- set(test.columns))
if test_overfit:
X.drop(test_overfit, axis=1, inplace=True)
 if target in X.columns:
X.drop([target], axis=1, inplace=True )<normalization>
test_df.to_csv('submission.csv', index=False )