Kaggle competition: Aerial Cactus Identification
<SOS> metric: AUC Kaggle data source: aerial-cactus-identification<categorify>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
dataset['AgeGroup'] = dataset['AgeGroup'].map({'boy': 0, 'girl': 1, 'adult male': 2, 'adult female': 3})
 dataset['AgeGroup'] = dataset['AgeGroup'].astype(int )<create_dataframe>
from fastai import *
from fastai.vision import *
pd.DataFrame(dataset[['Pclass', 'Survived', 'Sex']] 
.groupby(['Pclass', 'Sex', 'Survived'])
.apply(lambda x: x.shape[0]),
columns=['Count'] )<filter>
PATH = ".. /input/"
sz=32
bs=512
dataset[dataset['Embarked'].isnull() ]<create_dataframe>
tfms = get_transforms(flip_vert=True, max_rotate=90.)
data = ImageDataBunch.from_csv(PATH, ds_tfms=tfms,
folder="train/train", csv_labels='train.csv', test="test/test",
valid_pct=0.1, fn_col=0, label_col=1 ).normalize(imagenet_stats )
embarked_survived = pd.DataFrame(dataset[['Embarked', 'Survived']] 
.groupby(['Embarked', 'Survived'])
.apply(lambda x: x.shape[0]), columns=['count'] ).reset_index()
embarked_survived_pivot = embarked_survived.pivot(index='Embarked', columns='Survived', values='count')
 embarked_survived_pivot<count_values>
learn = create_cnn(data, models.resnet50, metrics=accuracy, path='.. /kaggle/working', model_dir='.. /kaggle/working/model',callback_fns=ShowGraph )
train['Embarked'].value_counts()<categorify>
lrf=learn.lr_find()
learn.recorder.plot()
dataset['Embarked'] = dataset['Embarked'].map({'S': 0, 'C': 1, 'Q': 2} )<data_type_conversions>
lr=5e-3
dataset['Embarked'].fillna(dataset['Embarked'].mode() [0], inplace=True )<data_type_conversions>
learn.fit_one_cycle(1,lr )
dataset['Embarked'] = dataset['Embarked'].astype('int' )<count_missing_values>
learn.save('cactus-stage-1' )
dataset['Embarked'].isnull().sum()<count_values>
lrf=learn.lr_find()
learn.recorder.plot()
dataset['Embarked'].value_counts()<filter>
learn.fit_one_cycle(3, max_lr=slice(1e-6, 1e-4))
dataset[dataset['Fare'].isnull() ]<filter>
learn.save('cactus-stage-2' )
old_man_nan_fare_idx = dataset[dataset['Fare'].isnull() ].index
old_man_nan_fare_idx<filter>
preds_test,y_test=learn.get_preds(ds_type=DatasetType.Test )
dataset[dataset['Pclass'] == 3][dataset['Cabin'].isnull() ][dataset['Age'] < 20]['Fare'].mean()<filter>
preds_test_tta,y_test_tta=learn.TTA(ds_type=DatasetType.Test )
mean_old_pclass3 = dataset[dataset['Pclass'] == 3][dataset['Cabin'].isnull() ][dataset['Age'] > 50]['Fare'].mean()
mean_old_pclass3<feature_engineering>
sub=pd.read_csv(f'{PATH}/sample_submission.csv' ).set_index('id' )
dataset.loc[old_man_nan_fare_idx, 'Fare'] = mean_old_pclass3<create_dataframe>
clean_fname=np.vectorize(lambda fname: str(fname ).split('/')[-1].split('.')[0]+".jpg")
 fname_cleaned=clean_fname(data.test_ds.items)
 fname_cleaned=fname_cleaned.astype(str)
 fname_cleaned
dataset.iloc[old_man_nan_fare_idx]<define_variables>
sub.loc[fname_cleaned,'has_cactus']=to_np(preds_test[:,1])
 sub.to_csv(f'submission.csv')
 sub.loc[fname_cleaned,'has_cactus']=to_np(preds_test_tta[:,1])
 sub.to_csv(f'submission_tta.csv' )
<categorify><EOS> 
classes = preds_test.argmax(1)
 classes
sub.loc[fname_cleaned,'has_cactus']=to_np(classes)
 sub.to_csv(f'submission_1_0.csv' )