Kaggle competition: Aerial Cactus Identification
<SOS> metric: AUC Kaggle data source: aerial-cactus-identification<split>
!pip install -q --upgrade pip
!pip install -q -U tensorflow-gpu==2.0.0-alpha0
get_from_id_tic2(have_family,need_more_check,['2506xx'] )<feature_engineering>
AUTOTUNE = tf.data.experimental.AUTOTUNE
have_family.loc[(have_family.PassengerId.isin([248,313,756,1041])) ,["Group_size","Ave_Survival"]] = [3,0.5]<split>
train_csv = pd.read_csv('.. /input/train.csv')
 train_csv.head()
get_from_id_tic2(have_family,need_more_check,['3470xx'] )<split>
display(Image('.. /input/train/train/0004be2cfeaba1c0361d39e2b000257b.jpg'))
display(Image('.. /input/train/train/000c8a36845c0208e833c79c1bffedd1.jpg'))
display(Image('.. /input/train/train/002134abf28af54575c18741b89dd2a4.jpg'))
display(Image('.. /input/train/train/0024320f43bdd490562246435af4f90b.jpg'))
get_from_id_tic2(have_family,need_more_check,['3500xx'] )<feature_engineering>
images = ['.. /input/train/train/' + fname for fname in train_csv['id']]
labels = train_csv['has_cactus'].tolist()
have_family.loc[(have_family.PassengerId.isin([193,722])) ,["Group_size","Ave_Survival"]] = [2,0.5]<feature_engineering>
X_train, X_dev, y_train, y_dev = train_test_split(images, labels, test_size=0.1, random_state=42)
 n_train = len(X_train)
 n_dev = len(X_dev )
have_family.loc[have_family.PassengerId==443,["Group_size",'Ave_Survival']] = [2,0]
have_family.loc[have_family.PassengerId==1106,["Group_size",'Ave_Survival']] = [7,0]<split>
IMAGE_SIZE = 96
def preprocess_image(fname, label=None):
img = tf.io.read_file(fname)
 img = tf.image.decode_jpeg(img)
 img = tf.cast(img, tf.float32)
 img =(img / 127.5)- 1
img = tf.image.resize(img, size=(IMAGE_SIZE, IMAGE_SIZE))
if label is not None:
return img, label
else:
return img
get_from_id_tic2(have_family,need_more_check,maybe_wrong )<feature_engineering>
BATCH_SIZE = 32
ds_train =(tf.data.Dataset.from_tensor_slices(( X_train, y_train)) 
.map(preprocess_image, num_parallel_calls=AUTOTUNE)
.shuffle(n_train)
.batch(BATCH_SIZE)
.prefetch(buffer_size=AUTOTUNE)
)
 ds_dev =(tf.data.Dataset.from_tensor_slices(( X_dev, y_dev)) 
.map(preprocess_image, num_parallel_calls=AUTOTUNE)
.shuffle(n_dev)
.batch(BATCH_SIZE)
.prefetch(buffer_size=AUTOTUNE)
 )
have_family.loc[have_family.PassengerId.isin([41,893,1296]),["SibSp","Family_size"]] = [0,1]
combine.loc[combine.PassengerId.isin([41,893,1296]),["SibSp","Family_size"]] = [0,1]
have_family.loc[have_family.PassengerId.isin([69,1106]),["Group_size",'Ave_Survival']] = [7,0.125]<feature_engineering>
base_model = tf.keras.applications.MobileNetV2(
 input_shape =(IMAGE_SIZE, IMAGE_SIZE, 3),
include_top = False,
weights = 'imagenet' 
)
 base_model.trainable = False
base_model.summary()
have_family = have_family[have_family.Family_size!=1]
passid_list = list(have_family.PassengerId.values)
 group_size_list = list(have_family.Group_size.values)
 ave_survival = list(have_family.Ave_Survival.values)
 combine.loc[combine.PassengerId.isin(passid_list),"Group_size"] = group_size_list
combine.loc[combine.PassengerId.isin(passid_list),"Ave_Survival"] = ave_survival<groupby>
pooling_layer = tf.keras.layers.GlobalMaxPooling2D()
final_layer = tf.keras.layers.Dense(units=1, activation='sigmoid' )
all_passenger = []
for i,j in combine[(combine.Surname_Counts>=2)].groupby(["Ticket2","Surname"]):
if len(j)>=2:
j_passid = list(j.PassengerId.values)
 all_passenger = all_passenger+j_passid
same_surname = list(combine[(combine.PassengerId.isin(all_passenger)) &(combine.Group_size.isna())]["PassengerId"].values)
 print(len(same_surname))<groupby>
model = tf.keras.Sequential([
base_model,
pooling_layer,
final_layer
] )
all_passenger = []
for i,j in combine[(combine.Maiden_Counts>=2)].groupby(["Ticket2","Maiden"]):
if len(j)>=2:
j_passid = list(j.PassengerId.values)
 all_passenger = all_passenger+j_passid
same_maiden = list(combine[(combine.PassengerId.isin(all_passenger)) &(combine.Group_size.isna())]["PassengerId"].values)
 print(same_maiden )<filter>
learning_rate = 0.0001
model.compile(
 optimizer=tf.optimizers.Adam(learning_rate=learning_rate),
loss='binary_crossentropy',
metrics=['accuracy']
)
combine[combine.Ticket=="31352"][['Age', 'Embarked', 'Parch', 'SibSp','Ticket','Surname', 'Maiden','Family_size','Pclass_Gender',
'Cabin_Title',"Surname_Counts","Group_size","Survived"]]<feature_engineering>
initial_epochs = 32
steps_per_epoch =(tf.math.ceil(n_train/BATCH_SIZE))
model.fit(
 ds_train.repeat() ,
epochs=initial_epochs,
steps_per_epoch=steps_per_epoch,
validation_data=ds_dev
)
team = combine[(combine.Ticket=="31352")|(( combine.Surname=="Ware")&(combine.Family_size!=1)) ]
combine.loc[(combine.PassengerId.isin([1170,1254])) ,"Group_size"] = 2
combine.loc[(combine.PassengerId.isin([1170,1254])) ,"Ave_Survival"] = team.Survived.mean()<create_dataframe>
base_model.trainable = True
len(base_model.layers )
same_surname.remove(1254)
 full_gs_and_as(dataset = combine,id_list=same_surname,ticket_kind = "Ticket2",check_list = [],with_friend=[],count_maiden = False )<filter>
fine_tune_at = 100
for layer in base_model.layers[:fine_tune_at]:
layer.trainable = False
combine[(combine.PassengerId.isin([115, 245, 303, 598, 693, 827])) ][["Pclass_Gender","Survived","Ticket"]]<feature_engineering>
model.compile(
 optimizer=tf.optimizers.Adam(learning_rate=learning_rate/10),
loss='binary_crossentropy',
metrics=['accuracy']
)
combine.loc[(combine.Ticket.isin(["2627","2694"])) ,"Group_size"] = combine[(combine.Ticket.isin(["2627","2694"])) ].shape[0]
combine.loc[(combine.Ticket.isin(["2627","2694"])) ,"Ave_Survival"] = combine[(combine.Ticket.isin(["2627","2694"])) ]["Survived"].mean()
combine.loc[(combine.Ticket.isin(["LINE"])) ,"Group_size"] = combine[(combine.Ticket.isin(["LINE"])) ].shape[0]
combine.loc[(combine.Ticket.isin(["LINE"])) ,"Ave_Survival"] = combine[(combine.Ticket.isin(["LINE"])) ]["Survived"].mean()
combine.loc[(combine.Ticket.isin(["1601"])) ,"Group_size"] = combine[(combine.Ticket.isin(["1601"])) ].shape[0]
combine.loc[(combine.Ticket.isin(["1601"])) ,"Ave_Survival"] = combine[(combine.Ticket.isin(["1601"])) ]["Survived"].mean()<define_variables>
fine_epochs = 32
total_epochs = initial_epochs + fine_epochs
model.fit(
 ds_train.repeat() ,
epochs=total_epochs,
initial_epoch=initial_epochs,
steps_per_epoch=steps_per_epoch,
validation_data=ds_dev
)
surname_list = list(combine.Surname.values)
 common_name = []
for maiden in combine.Maiden.values:
if maiden in surname_list:
common_name = common_name+[maiden]
print(len(common_name))
print(common_name[:5] )<groupby>
test_image_names = tf.io.gfile.listdir('.. /input/test/test/')
 n_test = len(test_image_names)
 test_image_paths = list(map(lambda s: '.. /input/test/test/' + s, test_image_names))
possible_cousin = combine[(( combine.Surname.isin(common_name)) |(combine.Maiden.isin(common_name)))
 &(combine.Family_size==1)&(combine.Group_size.isna())]
possible_cousin_id = []
for i,j in possible_cousin.groupby(["Ticket2"]):
if len(j)>=2:
j_passid = list(j.PassengerId.values)
 possible_cousin_id = possible_cousin_id+[j_passid]
print(possible_cousin_id )<filter>
ds_test =(tf.data.Dataset.from_tensor_slices(test_image_paths)
.map(preprocess_image, num_parallel_calls=AUTOTUNE)
.batch(BATCH_SIZE)
.prefetch(buffer_size=AUTOTUNE)
 )
combine[(combine.PassengerId.isin([41, 104, 714, 877])) ][["Surname","Maiden"]]<filter>
predictions = model.predict(ds_test )
combine[(combine.PassengerId.isin([41, 714])) ][['Age',"Cabin_Title", 'Embarked','Pclass_Gender',"Surname",'Maiden','Ticket',"Survived"]]<feature_engineering>
final_df = pd.DataFrame()
final_df['id'] = test_image_names
final_df['has_cactus'] = predictions
combine.loc[combine.PassengerId.isin([41, 714]),["Group_size",'Ave_Survival']] = [2,0]<categorify>
final_df.to_csv('submission.csv', index=False )