Kaggle competition: Aerial Cactus Identification
<SOS> metric: AUC Kaggle data source: aerial-cactus-identification<feature_engineering>
import numpy as np
import pandas as pd
from pathlib import Path
from fastai import *
from fastai.vision import *
import torchvision
import torch
team_id =get_team_id(have_family,["Olsen"])
 have_family.loc[have_family.PassengerId.isin(team_id),"Group_size"] = have_family[have_family.PassengerId.isin(team_id)].shape[0]
have_family.loc[have_family.PassengerId.isin(team_id),"Ave_Survival"] = have_family[have_family.PassengerId.isin(team_id)]["Survived"].mean()<feature_engineering>
data_root_path = Path(".. /input" )
team_id = get_team_id(have_family,["Crosby","Halstead"])
 have_family.loc[have_family.PassengerId.isin(team_id),"Group_size"] = have_family[have_family.PassengerId.isin(team_id)].shape[0]
have_family.loc[have_family.PassengerId.isin(team_id),"Ave_Survival"] = have_family[have_family.PassengerId.isin(team_id)]["Survived"].mean()<categorify>
train_df = pd.read_csv(data_root_path/"train.csv")
 test_df = pd.read_csv(data_root_path/"sample_submission.csv" )
def full_gs_and_as(dataset,id_list,ticket_kind,check_list,with_friend,count_maiden):
for i in id_list:
surname = set(dataset[dataset.PassengerId.isin([i])]["Surname"].values)
 ticket_list = set(dataset[dataset.PassengerId.isin([i])][ticket_kind].values)
 if count_maiden ==True:
maiden = set(dataset[dataset.PassengerId.isin([i])&(dataset.Maiden.notna())].Maiden.values)
 team = dataset[(( dataset.Surname.isin(surname|maiden))
|(dataset.Maiden.isin(surname|maiden)))
 &(dataset[ticket_kind].isin(ticket_list)) ]
team_pg = team.Pclass_Gender.describe().top
if ticket_kind=="Ticket":
team_size = team.shape[0]
ave_survival = dataset[dataset[ticket_kind].isin(ticket_list)].Survived.mean()
team_passid = list(team.PassengerId.values)
 else:
team_size = round(team.Family_size.describe().mean())
 ticket = set(team.Ticket.values)
 ave_survival = dataset[dataset.Ticket.isin(ticket)].Survived.mean()
team_passid = list(team.PassengerId.values)
 else:
team = dataset[(dataset[ticket_kind].isin(ticket_list)) &(dataset.Surname.isin(surname)) &(dataset.Group_size.isna())]
team_size = team.shape[0]
if team_size >=1:
team_pg = team.Pclass_Gender.describe().top
else:
team_pg = []
if team_size>=2:
ticket = set(team.Ticket.values)
 ticket_team_size = dataset[dataset.Ticket.isin(ticket)].shape[0]
if team_size == ticket_team_size:
team_passid = list(team.PassengerId.values)
 ave_survival=team.Survived.mean()
else:
team_passid = []
ave_survival = nan
with_friend = with_friend+[i]
else:
team_passid = []
ave_survival = nan
empty_survived = dataset[(dataset.PassengerId.isin(team_passid)) &(dataset.Survived.isna())].shape[0]
full_survived = dataset[(dataset.PassengerId.isin(team_passid)) &(dataset.Survived.notna())].shape[0]
if full_survived>0:
if empty_survived>0:
empty_gender_count = team[(team.Survived.isna())].Gender.describe() ["unique"]
empty_gender = set(team[(team.Survived.isna())].Gender.values)
 full_gender = set(team[(team.Survived.notna())].Gender.values)
 if ave_survival ==0:
if("Woman" in full_gender)|(( "Man" in empty_gender)&(empty_gender_count==1)) :
dataset.loc[dataset.PassengerId.isin(team_passid),"Ave_Survival"] = 0
else:
dataset.loc[dataset.PassengerId.isin(team_passid),"Ave_Survival"] = 0.5
elif ave_survival==1:
if("Man" in full_gender)|(( "Woman" in empty_gender)&(empty_gender_count==1)) :
dataset.loc[dataset.PassengerId.isin(team_passid),"Ave_Survival"] = 1
else:
dataset.loc[dataset.PassengerId.isin(team_passid),"Ave_Survival"] = 0.5
else:
check_list=check_list+[team_passid]
else:
dataset.loc[(dataset.PassengerId.isin(team_passid)) ,"Ave_Survival"]=ave_survival
else:
dataset.loc[(dataset.PassengerId.isin(team_passid)) ,"Ave_Survival"]=nan
dataset.loc[(dataset.PassengerId.isin(team_passid)) ,"Group_size"] = team_size
print("These passenger travel not only with cousin:",with_friend)
 check_list = list(np.unique(check_list))
print("these passenger need check more:",check_list )<create_dataframe>
transforms = get_transforms(
 do_flip=True,
flip_vert=True,
max_rotate=15.0,
max_lighting=0.15,
max_warp=0.2 
)
 train_imgs = ImageList.from_df(train_df, path=data_root_path/'train', folder='train')
 test_imgs = ImageList.from_df(test_df, path=data_root_path/'test', folder='test')
 train_imgs =(train_imgs 
.split_by_rand_pct(0.01)
.label_from_df() 
.add_test(test_imgs)
.transform(transforms, size=128)
.databunch(path='.', bs=64, device= torch.device('cuda:0')) 
.normalize(imagenet_stats))
full_gs_and_as(dataset = have_family,id_list=ship_team_2,ticket_kind = "Ticket",check_list = [],with_friend=[],count_maiden = True )<feature_engineering>
learn = cnn_learner(train_imgs, torchvision.models.densenet161, metrics=[error_rate, accuracy] )
def fillfull_p2(dataset,check_list,check_more):
for i in check_list:
pclass = dataset[dataset.PassengerId.isin(i)].Pclass.describe() ["mean"]
if pclass != 2:
check_more = check_more+[i]
else:
dataset.loc[dataset.PassengerId.isin(i),"Group_size"] = len(i)
 dataset.loc[dataset.PassengerId.isin(i),"Ave_Survival"] = dataset[dataset.PassengerId.isin(i)]["Survived"].mean()
return print(check_more )<train_model>
lr = 3e-02
learn.fit_one_cycle(8, slice(lr))
check_list = [[26, 183, 234, 262, 1046, 1066, 1271], [28, 89, 342, 439, 945, 961], [44, 609, 686, 1188], [59, 451, 473, 1142],
[94, 789, 924, 1246], [119, 300, 1076], [246, 413, 1303], [280, 747, 1284], [298, 306, 499, 1198], [671, 685, 1067],
[818, 828, 1253]]
fillfull_p2(dataset = have_family,check_list = check_list,check_more = [] )<categorify>
preds,_ = learn.get_preds(ds_type=DatasetType.Test )
def check_ave_survival(dataset,passid):
return dataset[dataset.PassengerId.isin(passid)][["Pclass_Gender","Survived"]]
def fullfill_ave_survival(dataset,check_list,check_more):
for i in check_list:
ave_survival = dataset[dataset.PassengerId.isin(i)].Survived.mean()
empty_gender = list(dataset[(dataset.PassengerId.isin(i)) &(dataset.Survived.isna())].Gender.values)
 if ave_survival>0.5:
if "Man" in empty_gender:
dataset.loc[(dataset.PassengerId.isin(i)) ,"Ave_Survival"] = 0.5
else:
dataset.loc[(dataset.PassengerId.isin(i)) ,"Ave_Survival"] = ave_survival
elif ave_survival==0.5:
if "Woman"in empty_gender:
dataset.loc[(dataset.PassengerId.isin(i)) ,"Ave_Survival"] = 0.5
else:
dataset.loc[(dataset.PassengerId.isin(i)) ,"Ave_Survival"] = ave_survival
else:
check_more = check_more+[i]
return print(check_more )<compute_train_metric>
test_df.has_cactus = preds.numpy() [:, 0]
<feature_engineering><EOS> 
test_df.to_csv('submission.csv', index=False )