Kaggle competition: Aerial Cactus Identification
train.loc[(train.Fare==0), 'Fare'] = train.Fare.median()
test.loc[(test.Fare==0), 'Fare'] = test.Fare.median()
test.loc[(test.Fare.isnull()), 'Fare'] = test.Fare.median()<feature_engineering>
%reload_ext autoreload
%autoreload 2
%matplotlib inline
train['age_log'] = np.log(train['Age'])
 test['age_log'] = np.log(test['Age'])
 train['fare_log'] = np.log(train['Fare'])
 test['fare_log'] = np.log(test['Fare'] )<normalization>
from fastai import *
from fastai.vision import *
scaler = StandardScaler()
scaler.fit(train[['Age']])
 train['age_scale'] = scaler.transform(train[['Age']])
 test['age_scale'] = scaler.transform(test[['Age']])
 scaler = StandardScaler()
scaler.fit(train[['Fare']])
 train['fare_scale'] = scaler.transform(train[['Fare']])
 test['fare_scale'] = scaler.transform(test[['Fare']] )<categorify>
bs = 64
train_objs_num = len(train)
 dataset = pd.concat(objs=[train, test],sort=False)
 dataset['ngroup'] = dataset.groupby(['Pclass','Sex'] ).ngroup()
dataset = dataset['ngroup']
train['ngroup'] = copy.copy(dataset[:train_objs_num])
 test['ngroup'] = copy.copy(dataset[train_objs_num:] )<categorify>
path = Path('.. /input')
 path_train = path/'train/train'
path_test = path/'test/test/'
path, path_train, path_test
train['SibParchSum'] = train['SibSp'] + train['Parch']
test['SibParchSum'] = test['SibSp'] + test['Parch']
train['IsWithFamily'] =(train['SibParchSum']>0 ).astype(int)
 test['IsWithFamily'] =(test['SibParchSum']>0 ).astype(int)
 binner = KBinsDiscretizer(encode='ordinal')
 binner.fit(train[['Fare']])
 train['Fare_bins'] = binner.transform(train[['Fare']])
 test['Fare_bins'] = binner.transform(test[['Fare']])
 binner = KBinsDiscretizer(encode='ordinal')
 binner.fit(train[['Age']])
 train['age_bins'] = binner.transform(train[['Age']])
 test['age_bins'] = binner.transform(test[['Age']] )<categorify>
labels_df = pd.read_csv(path/'train.csv')
 test_df = pd.read_csv(path/'sample_submission.csv')
 labels_df.head()
encoder = OrdinalEncoder()
encoder.fit(train[['Sex']])
 train['sex_integer'] = encoder.transform(train[['Sex']])
 test['sex_integer'] = encoder.transform(test[['Sex']])
 encoder = OrdinalEncoder()
encoder.fit(train[['Embarked']])
 train['embarked_integer'] = encoder.transform(train[['Embarked']])
 test['embarked_integer'] = encoder.transform(test[['Embarked']] )<categorify>
np.random.seed(42)
 test = ImageList.from_df(test_df, path=path_test)
 data =(
 ImageList.from_df(labels_df, path=path_train)
.split_by_rand_pct(0.01)
.label_from_df() 
.add_test(test)
.transform(get_transforms(
 flip_vert = True, 
), size = 128)
.databunch(path=path, bs = bs ).normalize(imagenet_stats)
 )
train_after_normalizer = preprocessing.Normalizer().fit_transform(train[['Age', 'Fare','SibSp', 'Parch','sex_integer','embarked_integer','Pclass']] )<create_dataframe>
data.show_batch(rows = 3, figsize =(10,8))
train_after_normalizer = pd.DataFrame(train_after_normalizer, columns = ['Age', 'Fare','SibSp', 'Parch','sex_integer','embarked_integer','Pclass'] )<groupby>
learn = cnn_learner(data, models.resnet101, metrics = accuracy, model_dir='/tmp/model/' )
print(train.groupby('Pclass')['Fare'].mean())
 print(test.groupby('Pclass')['Fare'].mean())
 print(test[test.Fare.isnull() ])
<feature_engineering>
learn.lr_find()
test.loc[test.Fare.isnull() ,'Fare'] = 13<filter>
lr = 3e-02
test.loc[test.PassengerId==1044]<groupby>
learn.fit_one_cycle(3, slice(lr))
test.groupby(['Pclass','Sex'])['Age'].mean()<count_values>
learn.save('resnet-101-1' )
train['Survived'].value_counts(sort = False )<count_missing_values>
learn = cnn_learner(data, models.densenet161, metrics = accuracy, model_dir='/tmp/model/' )
total = train.isnull().sum().sort_values(ascending=False)
 percent =(train.isnull().sum() /train.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data.head(5 )<sort_values>
learn.lr_find()
total = test.isnull().sum().sort_values(ascending=False)
 percent =(test.isnull().sum() /test.isnull().count() ).sort_values(ascending=False)
 missing_test_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_test_data.head(5 )<count_unique_values>
lr = 3e-02
train.nunique()<prepare_x_and_y>
learn.fit_one_cycle(3, slice(lr))
y_train = train.Survived<groupby>
learn.save('densenet-161-1' )
train.groupby(train.Cabin.isnull() , as_index=False ).size()<count_values>
learn = cnn_learner(data, models.resnet101, metrics = accuracy, model_dir='/tmp/model/' )
train['SibSp'].value_counts(sort=False )<groupby>
learn.load('resnet-101-1');
train[['SibSp', 'Survived']].groupby('SibSp')['Survived'].sum()<groupby>
preds, _ = learn.get_preds(ds_type=DatasetType.Test )
( train[['SibSp', 'Survived']].groupby('SibSp')['Survived'].sum() /train['SibSp'].value_counts(sort=False)) *100<count_values>
test_df['has_cactus'] = np.array(preds[:, 0])
 test_df.head()
train.Parch.value_counts(sort=False )<groupby>
test_df.to_csv('submission_resnet_101.csv', index = False )
train[['Parch', 'Survived']].groupby('Parch')['Survived'].sum()<groupby>
learn = cnn_learner(data, models.densenet161, metrics = accuracy, model_dir='/tmp/model/' )
( train[['Parch', 'Survived']].groupby('Parch')['Survived'].sum() /train.Parch.value_counts(sort=False)) *100<groupby>
learn.load('densenet-161-1');
( train[['Pclass', 'Survived']].groupby('Pclass')['Survived'].sum() /train.Pclass.value_counts(sort=False)) *100<count_values>
preds, _ = learn.get_preds(ds_type=DatasetType.Test )
train.Embarked.value_counts()<count_unique_values>
test_df['has_cactus'] = np.array(preds[:, 0])
 test_df.head()
const_cols = [c for c in train.columns if train[c].nunique(dropna=False)==1 ]
const_cols<groupby>
test_df.to_csv('submission_densenet_161.csv', index = False )
train.groupby('Pclass')['Age'].agg(['size', 'count', 'mean'] )<groupby>
FileLinks('.' )