Kaggle competition: Aerial Cactus Identification
def detect_outliers(df,n,features):

 outlier_indices = []
for col in features:
Q1 = np.percentile(df[col], 25)
 Q3 = np.percentile(df[col],75)
 IQR = Q3 - Q1
outlier_step = 1.7 * IQR
outlier_list_col = df[(df[col] < Q1 - outlier_step)|(df[col] > Q3 + outlier_step)].index
outlier_indices.extend(outlier_list_col)
 outlier_indices = Counter(outlier_indices)
 multiple_outliers = list(k for k, v in outlier_indices.items() if v > n)
 return multiple_outliers
Outliers_to_drop = detect_outliers(train,2,["Age","SibSp","Parch","Fare"])
 train.loc[Outliers_to_drop]<concatenate>
train_csv = pd.read_csv(path/'train.csv')
 test_df = pd.read_csv(path/'sample_submission.csv')
 test_data = ImageList.from_df(test_df, path=path/'test', folder='test')
 data = ImageDataBunch.from_csv(path, csv_labels='train.csv', folder='train/train', size=32)
 data.add_test(test_data)
 data.normalize(imagenet_stats)
 data.show_batch(rows=3, figsize=(5,5))
df = pd.concat(( train.loc[:,'Pclass':'Embarked'], test.loc[:,'Pclass':'Embarked'])).reset_index(drop=True )<drop_column>
learn = cnn_learner(data, models.resnet34, metrics=error_rate, model_dir=model_path )
survived = train.drop(train[train['Survived'] != 1].index)
 not_survived = train.drop(train[train['Survived'] != 0].index)
 basic_analysis(survived,not_survived )<feature_engineering>
learn.fit_one_cycle(4 )
def basic_details(df):
b = pd.DataFrame()
b['Missing value, %'] = round(df.isnull().sum() /df.shape[0]*100)
 b['N unique value'] = df.nunique()
b['dtype'] = df.dtypes
return b
basic_details(df )<categorify>
learn.save('first' )
df['Title'] = df.Name.str.extract('([A-Za-z]+)\.', expand=False)
 df['Title'] = df['Title'].replace(['Lady', 'Countess','Capt', 'Col', 'Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 df['Title'] = df['Title'].replace('Mlle', 'Miss')
 df['Title'] = df['Title'].replace('Ms', 'Miss')
 df['Title'] = df['Title'].replace('Mme', 'Mrs')
 title_mapping = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
df['Title'] = df['Title'].map(title_mapping)
 df['Title'] = df['Title'].fillna(0)
 df = df.drop(['Name'], axis=1)
 df["Sex"][df["Sex"] == "male"] = 0
df["Sex"][df["Sex"] == "female"] = 1
df["Sex"] = df["Sex"].astype(int)
 df['Age'] = df.groupby(['Pclass'])['Age'].transform(lambda x: x.fillna(x.median()))
df["Age"] = df["Age"].astype(int)
 df['Age_cat'] = pd.qcut(df['Age'],q=[0,.16,.33,.49,.66,.83, 1], labels=False, precision=1)
 tickets = []
for i in list(df.Ticket):
if not i.isdigit() :
tickets.append(i.replace(".","" ).replace("/","" ).strip().split(" ")[0])
 else:
tickets.append("x")
 df["Ticket"] = tickets
df['Ticket_Frequency'] = df.groupby('Ticket')['Ticket'].transform('count')
 df = pd.get_dummies(df, columns= ["Ticket"], prefix = "T")
 df['Fare'] = df.groupby("Pclass")['Fare'].transform(lambda x: x.fillna(x.median()))
df['Zero_Fare'] = df['Fare'].map(lambda x: 1 if x == 0 else(0))
def fare_category(fr):
if fr <= 7.91:
return 1
elif fr <= 14.454 and fr > 7.91:
return 2
elif fr <= 31 and fr > 14.454:
return 3
return 4
df['Fare_cat'] = df['Fare'].apply(fare_category)
 df['Cabin'] = df['Cabin'].fillna('U')
 df['Cabin'] = df['Cabin'].map(lambda x: re.compile("([a-zA-Z]+)" ).search(x ).group())
 cabin_category = {'A':9, 'B':8, 'C':7, 'D':6, 'E':5, 'F':4, 'G':3, 'T':2, 'U':1}
df['Cabin'] = df['Cabin'].map(cabin_category)
 df["Embarked"] = df["Embarked"].fillna("C")
 df["Embarked"][df["Embarked"] == "S"] = 1
df["Embarked"][df["Embarked"] == "C"] = 2
df["Embarked"][df["Embarked"] == "Q"] = 3
df["Embarked"] = df["Embarked"].astype(int)
 df['FamilySize'] = df['SibSp'] + df['Parch'] + 1
df['FamilySize_cat'] = df['FamilySize'].map(lambda x: 1 if x == 1
else(2 if 5 > x >= 2
else(3 if 8 > x >= 5
else 4)
))
df['Alone'] = [1 if i == 1 else 0 for i in df['FamilySize']]
dummy_col=['Title', 'Sex', 'Age_cat', 'SibSp', 'Parch', 'Fare_cat', 'Embarked', 'Pclass', 'FamilySize_cat']
dummy = pd.get_dummies(df[dummy_col], columns=dummy_col, drop_first=False)
 df = pd.concat([dummy, df], axis = 1)
 dummy_fare = ['Fare']
dummy_f = pd.get_dummies(df[dummy_fare], columns=dummy_fare, drop_first=True)
 df = pd.concat([dummy_f, df], axis = 1)
 df['FareCat_Sex'] = df['Fare_cat']*df['Sex']
df['Pcl_Sex'] = df['Pclass']*df['Sex']
df['Pcl_Title'] = df['Pclass']*df['Title']
df['Age_cat_Sex'] = df['Age_cat']*df['Sex']
df['Age_cat_Pclass'] = df['Age_cat']*df['Pclass']
df['Title_Sex'] = df['Title']*df['Sex']
df['SmallF'] = df['FamilySize'].map(lambda s: 1 if s == 2 else 0)
 df['MedF'] = df['FamilySize'].map(lambda s: 1 if 3 <= s <= 4 else 0)
 df['LargeF'] = df['FamilySize'].map(lambda s: 1 if s >= 5 else 0)
 df['Senior'] = df['Age'].map(lambda s:1 if s>70 else 0)
 df["Fare"] = df["Fare"].map(lambda i: np.log(i)if i > 0 else 0 )<data_type_conversions>
learn.load('first')
 learn.unfreeze()
learn.fit_one_cycle(3, max_lr=5e-5 )
def des_stat_feat(df):
df = pd.DataFrame(df)
 dcol= [c for c in df.columns if df[c].nunique() >=10]
d_median = df[dcol].median(axis=0)
 d_mean = df[dcol].mean(axis=0)
 q1 = df[dcol].apply(np.float32 ).quantile(0.25)
 q3 = df[dcol].apply(np.float32 ).quantile(0.75)
 for c in dcol:
df[c+str('_median_range')] =(df[c].astype(np.float32 ).values > d_median[c] ).astype(np.int8)
 df[c+str('_mean_range')] =(df[c].astype(np.float32 ).values > d_mean[c] ).astype(np.int8)
 df[c+str('_q1')] =(df[c].astype(np.float32 ).values < q1[c] ).astype(np.int8)
 df[c+str('_q3')] =(df[c].astype(np.float32 ).values > q3[c] ).astype(np.int8)
 return df 
<feature_engineering>
preds, _ = learn.get_preds(ds_type=DatasetType.Test )
<count_duplicates><EOS> 
test_df = pd.read_csv(path/'sample_submission.csv')
 test_df.has_cactus = preds.numpy() [:, 0]
test_df.to_csv('submission.csv', index=False )