Kaggle competition: Aerial Cactus Identification
path = '.. /input/house-prices-advanced-regression-techniques/'
X_full = pd.read_csv(path + 'train.csv', index_col=0)
 test = pd.read_csv(path + 'test.csv', index_col=0)
 target = 'SalePrice'
y = X_full[target]
X = X_full.drop(target, axis=1)
 dataset = [X, test]
X.head()
SEED = 2007<categorify>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
def apply(dfs, func, *largs, **kwargs):
for df in dfs:
df = func(df, *largs, **kwargs)
 return dfs<normalization>
from fastai.vision import *
import fastai
def norm(df):
return np.log1p(df)
 def un_norm(df):
return np.expm1(df)
 y = norm(y)
 plot_dist_prob(y, 'Normalized')
 X[target] = y<drop_column>
path = Path('.. /input' )
def drop(df, remove, axis=0):
df.drop(remove, axis=axis, inplace=True)
 return df
mean = y.mean()
std = y.std()
outliers =(y > mean + 3 * std)|(y < mean - 3 * std)
 outliers = outliers[outliers].index
un_norm(y[outliers])
 X = drop(X, outliers, axis=0)
 y = drop(y, outliers, axis=0)
<categorify>
train_df = pd.read_csv(path/'train.csv')
 train_df.head()
def make_categorical(dfs):
col_transform = ['MSSubClass', 'MoSold']
for col in col_transform:
for df in dfs:
df[col] = df[col].apply(str)
 return dfs
dataset = make_categorical(dataset )<drop_column>
test_df = pd.read_csv(path/'sample_submission.csv')
 print(test_df.shape)
 test_df.head()
def make_drop(df, cols):
return df.drop(cols, axis=1, inplace=True)
 def drop_problematic(dfs):
problem_cols = ['MasVnrType',
'MasVnrArea',
'MiscFeature']
for df in dfs:
df = make_drop(df, problem_cols)
 return dfs
dataset = drop_problematic(dataset )<categorify>
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=10.0, max_zoom=1.1, max_lighting=0.2, max_warp=0.2, p_affine=0.75, p_lighting=0.75)
 train_data = ImageDataBunch.from_df(path/'train'/'train', train_df, ds_tfms=tfms, size=128 )
def make_impute(df):
df['GarageYrBlt'].fillna(df['YearBuilt'], inplace=True)
 df['Exterior1st'].fillna('Other', inplace=True)
 df['Exterior2nd'].fillna('Other', inplace=True)
 mode_cols = ['MSZoning', 'Electrical', 'SaleType']
for col in mode_cols:
mode = X[col].mode() [0]
df[col].fillna(mode, inplace=True)
 zero_cols = ['BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF',
'TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath',
'GarageCars', 'LotFrontage']
for col in zero_cols:
df[col].fillna(0, inplace=True)
 return df
def impute(dfs):
for df in dfs:
df = make_impute(df)
 return dfs
dataset = impute(dataset )<categorify>
train_data.show_batch(rows=3, figsize=(5,6))
def categorical_to_meaning(dfs):
col_order = [ 
('Alley', ['', 'Grvl', 'Pave']), 
('Utilities', ['', 'ELO', 'NoSeWa', 'NoSewr', 'AllPub']), 
('LotShape', ['Reg', 'IR1', 'IR2', 'IR3']), 
('LandContour', ['Lvl', 'Bnk', 'HLS', 'Low']), 
('LandSlope', ['Gtl', 'Mod', 'Sev']), 
('Fence', ['NA', 'MnWw', 'GdWo', 'MnPrv', 'GdPrv']), 
('PoolQC', ['NA', 'Fa', 'Gd', 'Ex']), 
('PavedDrive', ['N', 'P', 'Y']), 
('GarageCond', ['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex']), 
('GarageQual', ['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex']), 
('GarageFinish', ['NA', 'Unf', 'RFn', 'Fin']), 
('GarageType', ['NA', 'Detchd', 'CarPort', 'BuiltIn',
'Basment', 'Attchd', '2Types']), 
('FireplaceQu', ['NA', 'Po', 'Fa', 'TA', 'Gd', 'Ex']), 
('Functional', ['Sal', 'Sev', 'Maj2', 'Maj1', 'Mod',
'Min2', 'Min1', 'Typ']), 
('KitchenQual', ['', 'Po', 'Fa', 'TA', 'Gd', 'Ex']), 
('HeatingQC', ['', 'Po', 'Fa', 'TA', 'Gd', 'Ex']), 
('BsmtFinType1', ['', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ']), 
('BsmtFinType2', ['', 'Unf', 'LwQ', 'Rec', 'BLQ', 'ALQ', 'GLQ']), 
('BsmtExposure', ['', 'No', 'Mn', 'Av', 'Gd']), 
('BsmtCond', ['', 'Po', 'Fa', 'TA', 'Gd', 'Ex']), 
('BsmtQual', ['', 'Po', 'Fa', 'TA', 'Gd', 'Ex']), 
('Foundation', ['Wood', 'Stone', 'Slab', 'PConc', 'CBlock', 'BrkTil']), 
('ExterCond', ['', 'Po', 'Fa', 'TA', 'Gd', 'Ex']), 
('ExterQual', ['', 'Po', 'Fa', 'TA', 'Gd', 'Ex']), 
('CentralAir', ['N', 'Y']), 
('Street', ['Grvl', 'Pave']),
]
for co, order in col_order:
imp = order[0]
d = {v: i for i, v in enumerate(order)}
for df in dfs:
df[co] = df[co].fillna(imp ).apply(lambda x: d.get(x, x))
return dfs
dataset = categorical_to_meaning(dataset)
<count_values>
learn = cnn_learner(train_data, models.densenet161, metrics=[accuracy],model_dir="/tmp/model/" )
nans = test['MSZoning'].isna()
test[nans]
X['Electrical'].mode() [0]
X['Electrical'].value_counts()<feature_engineering>
learn.lr_find()
def has_features(dfs):
have = ['GarageArea', 'PoolArea',
'TotalBsmtSF', '2ndFlrSF',
]
for df in dfs:
for col in have:
df['Has'+col] = df[col] > 0
df['has_remod'] = df['YearRemodAdd'] != df['YearBuilt']
df['is_new'] = df['YrSold'] == df['YearBuilt']
return dfs
dataset = has_features(dataset )<drop_column>
lr =1.0e-2
learn.fit_one_cycle(7,slice(lr))
twins_col = ['Fence', 'Fireplaces', 'FireplaceQu',
'PoolArea', 'PoolQC', 'GarageQual',
'GarageArea','GarageCond', '2ndFlrSF',
'BsmtCond', 'TotRmsAbvGrd','GarageFinish',
]
X.drop(twins_col, axis=1, inplace=True)
 test.drop(twins_col, axis=1, inplace=True )<categorify>
solution = pd.DataFrame(columns=test_df.columns)
 solution
C1, C2 = cols = ['Condition1', 'Condition2']
def condition_weight(val, weights):
return weights.get(val, 0)
 def extract_condition(df, new_features):
for col, wei in new_features.items() :
df[col] = df[C1].apply(
 lambda x: condition_weight(x, wei)
)+ df[C2].apply(
 lambda x: condition_weight(x, wei)
)
 df.drop(cols, axis=1, inplace=True)
 return df
def conditions(dfs):
artery = {'Feedr': 1, 'Artery': 2}
rail = {'RRNe': -2, 'RRNe': -2, 'RRAe': -1, 'RRAn': -1}
posi = {'PosN': 1, 'PosA': 2}
new_features = {
'Artery': artery,
'RailRoad': rail,
'Positive': posi,
}
for df in dfs:
df = extract_condition(df, new_features)
 return dfs
dataset = conditions(dataset )<feature_engineering>
for index,row in test_df.iterrows() :
img_name = row['id']
img = open_image(path/'test'/'test'/img_name)
 pred_class,pred_idx,outputs = learn.predict(img)
 solution.loc[len(solution)] = [img_name,outputs.numpy() [1]]
<feature_engineering><EOS> 
solution.to_csv('submission.csv', index=False )