Kaggle competition: Airbus Ship Detection Challenge
<SOS> metric: IntersectionOverUnionObjectSegmentationBeta Kaggle data source: airbus-ship-detection<count_missing_values>
from fastai.conv_learner import *
from fastai.dataset import *
import pandas as pd
import numpy as np
import os
from PIL import Image
from sklearn.model_selection import train_test_split
from tqdm import tnrange, tqdm_notebook
from scipy import ndimage
y.isna().sum()<categorify>
PATH = './'
TRAIN = '.. /input/airbus-ship-detection/train_v2/'
TEST = '.. /input/airbus-ship-detection/test_v2/'
SEGMENTATION = '.. /input/airbus-ship-detection/train_ship_segmentations_v2.csv'
PRETRAINED_DETECTION_PATH = '.. /input/fine-tuning-resnet34-on-ship-detection/models/'
PRETRAINED_SEGMENTATION_PATH = '.. /input/unet34-dice-0-87/models/'
DETECTION_TEST_PRED = '.. /input/fine-tuning-resnet34-on-ship-detection-new-data/ship_detection.csv'
X = X / 255.0
X = X.values.reshape(-1, 28, 28, 1)
 y = to_categorical(y, num_classes = 10 )<split>
nw = 2
arch = resnet34
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42 )<choose_model_class>
train_names = [f for f in os.listdir(TRAIN)]
test_names = [f for f in os.listdir(TEST)]
tr_n, val_n = train_test_split(train_names, test_size=0.05, random_state=42)
 segmentation_df = pd.read_csv(os.path.join(PATH, SEGMENTATION)).set_index('ImageId' )
cnn_model = Sequential()
cnn_model.add(Conv2D(filters = 32, kernel_size =(5, 5), padding = 'same', activation ='relu', input_shape =(28,28,1)))
 cnn_model.add(Conv2D(filters = 32, kernel_size =(5, 5), padding = 'same', activation ='relu'))
cnn_model.add(MaxPool2D(pool_size=(2, 2)))
 cnn_model.add(Dropout(0.25))
cnn_model.add(Conv2D(filters = 32, kernel_size =(5, 5), padding = 'same', activation ='relu'))
cnn_model.add(Conv2D(filters = 32, kernel_size =(5, 5), padding = 'same', activation ='relu'))
cnn_model.add(MaxPool2D(pool_size=(2, 2), strides=(2, 2)))
 cnn_model.add(Dropout(0.25))
cnn_model.add(Conv2D(filters = 64, kernel_size =(3, 3), padding = 'same', activation ='relu'))
cnn_model.add(Conv2D(filters = 64, kernel_size =(3, 3), padding = 'same', activation ='relu'))
cnn_model.add(MaxPool2D(pool_size=(2, 2), strides=(2, 2)))
 cnn_model.add(Dropout(0.25))
cnn_model.add(Flatten())
 cnn_model.add(Dense(256, activation = "relu"))
cnn_model.add(Dropout(0.5))
cnn_model.add(Dense(10, activation = "softmax"))
print(cnn_model.summary() )<choose_model_class>
def cut_empty(names):
return [name for name in names
if(type(segmentation_df.loc[name]['EncodedPixels'])!= float)]
tr_n_cut = cut_empty(tr_n)
 val_n_cut = cut_empty(val_n )
cnn_model.compile(
 optimizer = 'adam',
loss = "categorical_crossentropy",
metrics=["accuracy"]
)<choose_model_class>
def get_mask(img_id, df):
shape =(768,768)
 img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 masks = df.loc[img_id]['EncodedPixels']
if(type(masks)== float): return img.reshape(shape)
 if(type(masks)== str): masks = [masks]
for mask in masks:
s = mask.split()
for i in range(len(s)//2):
start = int(s[2*i])- 1
length = int(s[2*i+1])
 img[start:start+length] = 1
return img.reshape(shape ).T
early_stopping = keras.callbacks.EarlyStopping(
 patience=5,
min_delta=0.001,
restore_best_weights=True,
)<train_model>
class pdFilesDataset(FilesDataset):
def __init__(self, fnames, path, transform):
self.segmentation_df = pd.read_csv(SEGMENTATION ).set_index('ImageId')
 super().__init__(fnames, transform, path)
 def get_x(self, i):
img = open_image(os.path.join(self.path, self.fnames[i]))
if self.sz == 768: return img
else: return cv2.resize(img,(self.sz, self.sz))
def get_y(self, i):
mask = np.zeros(( 768,768), dtype=np.uint8)if(self.path == TEST)\
else get_mask(self.fnames[i], self.segmentation_df)
 img = Image.fromarray(mask ).resize(( self.sz, self.sz)).convert('RGB')
 return np.array(img ).astype(np.float32)
 def get_c(self): return 0
datagen = ImageDataGenerator(
 rotation_range=10,
width_shift_range=0.2,
height_shift_range=0.2,
zoom_range = 0.1, 
)
 datagen.fit(X_train )<train_model>
def get_data(sz,bs):
tfms = tfms_from_model(arch, sz, crop_type=CropType.NO, tfm_y=TfmType.CLASS)
 tr_names = tr_n if(len(tr_n_cut)%bs == 0)else tr_n[:-(len(tr_n_cut)%bs)]
ds = ImageData.get_ds(pdFilesDataset,(tr_names,TRAIN), 
(val_n_cut,TRAIN), tfms, test=(test_names,TEST))
md = ImageData(PATH, ds, bs, num_workers=nw, classes=None)
 return md
history = cnn_model.fit_generator(
 datagen.flow(X_train,y_train, batch_size=64),
validation_data=(X_test, y_test),
steps_per_epoch=X_train.shape[0] // 64,
epochs=30,
callbacks=[early_stopping]
)<predict_on_test>
cut,lr_cut = model_meta[arch]
test_data = test_data / 255.0
test_data = test_data.values.reshape(-1, 28, 28, 1)
 results = cnn_model.predict(test_data)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )<save_to_csv>
def get_base(pre=True):
layers = cut_model(arch(pre), cut)
 return nn.Sequential(*layers )
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )<import_modules>
class UnetBlock(nn.Module):
def __init__(self, up_in, x_in, n_out):
super().__init__()
up_out = x_out = n_out//2
self.x_conv = nn.Conv2d(x_in, x_out, 1)
 self.tr_conv = nn.ConvTranspose2d(up_in, up_out, 2, stride=2)
 self.bn = nn.BatchNorm2d(n_out)
 def forward(self, up_p, x_p):
up_p = self.tr_conv(up_p)
 x_p = self.x_conv(x_p)
 cat_p = torch.cat([up_p,x_p], dim=1)
 return self.bn(F.relu(cat_p))
class SaveFeatures() :
features=None
def __init__(self, m): self.hook = m.register_forward_hook(self.hook_fn)
 def hook_fn(self, module, input, output): self.features = output
def remove(self): self.hook.remove()
class Unet34(nn.Module):
def __init__(self, rn):
super().__init__()
self.rn = rn
self.sfs = [SaveFeatures(rn[i])for i in [2,4,5,6]]
self.up1 = UnetBlock(512,256,256)
 self.up2 = UnetBlock(256,128,256)
 self.up3 = UnetBlock(256,64,256)
 self.up4 = UnetBlock(256,64,256)
 self.up5 = nn.ConvTranspose2d(256, 1, 2, stride=2)
 def forward(self,x):
x = F.relu(self.rn(x))
x = self.up1(x, self.sfs[3].features)
 x = self.up2(x, self.sfs[2].features)
 x = self.up3(x, self.sfs[1].features)
 x = self.up4(x, self.sfs[0].features)
 x = self.up5(x)
 return x[:,0]
def close(self):
for sf in self.sfs: sf.remove()
class UnetModel() :
def __init__(self,model,name='Unet'):
self.model,self.name = model,name
def get_layer_groups(self, precompute):
lgs = list(split_by_idxs(children(self.model.rn), [lr_cut]))
return lgs + [children(self.model)[1:]]
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from tensorflow import keras
from keras.utils.np_utils import to_categorical
from tensorflow.keras.applications.resnet50 import preprocess_input
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten, Conv2D, Dropout, MaxPool2D<load_from_csv>
def IoU(pred, targs):
pred =(pred > 0.5 ).astype(float)
 intersection =(pred*targs ).sum()
return intersection /(( pred+targs ).sum() - intersection + 1.0 )
righe, colonne = 28,28
n_classi = 10
test = pd.read_csv(".. /input/digit-recognizer/test.csv")
 train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 y_train = train["label"]
X_train = train.drop(labels = ["label"],axis = 1)
 X_train = X_train / 255.0
test = test / 255.0
X_train = X_train.values.reshape(-1,righe,colonne,1)
 test = test.values.reshape(-1,righe,colonne,1)
 y_train = to_categorical(y_train, num_classes = n_classi)
 plt.figure(figsize=(15,4.5))
for i in range(30):
plt.subplot(3, 10, i+1)
 plt.imshow(X_train[i].reshape(( righe,colonne)) ,cmap=plt.cm.binary)
 plt.axis('off')
 plt.subplots_adjust(wspace=-0.1, hspace=-0.1)
 plt.show()<choose_model_class>
def get_score(pred, true):
n_th = 10
b = 4
thresholds = [0.5 + 0.05*i for i in range(n_th)]
n_masks = len(true)
 n_pred = len(pred)
 ious = []
score = 0
for mask in true:
buf = []
for p in pred: buf.append(IoU(p,mask))
ious.append(buf)
 for t in thresholds:
tp, fp, fn = 0, 0, 0
for i in range(n_masks):
match = False
for j in range(n_pred):
if ious[i][j] > t: match = True
if not match: fn += 1
for j in range(n_pred):
match = False
for i in range(n_masks):
if ious[i][j] > t: match = True
if match: tp += 1
else: fp += 1
score +=(( b+1)*tp)/(( b+1)*tp + b*fn + fp)
 return score/n_th
datagen = ImageDataGenerator(
 rotation_range=10,
zoom_range = 0.10,
width_shift_range=0.1,
height_shift_range=0.1 )<choose_model_class>
def split_mask(mask):
threshold = 0.5
threshold_obj = 30
labled,n_objs = ndimage.label(mask > threshold)
 result = []
for i in range(n_objs):
obj =(labled == i + 1 ).astype(int)
 if(obj.sum() > threshold_obj): result.append(obj)
 return result
nets = 7
model = [0] *nets
for i in range(nets):
model[i] = Sequential()
model[i].add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(righe,colonne,1)))
 model[i].add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
model[i].add(MaxPool2D(pool_size=(2,2)))
 model[i].add(Dropout(0.25))
model[i].add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model[i].add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model[i].add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model[i].add(Dropout(0.25))
model[i].add(Flatten())
 model[i].add(Dense(256, activation = "relu"))
model[i].add(Dropout(0.5))
model[i].add(Dense(n_classi, activation = "softmax"))
model[i].compile(loss="categorical_crossentropy",
optimizer='adam',
metrics=['accuracy'] )<train_model>
def get_mask_ind(img_id, df, shape =(768,768)) :
masks = df.loc[img_id]['EncodedPixels']
if(type(masks)== float): return []
if(type(masks)== str): masks = [masks]
result = []
for mask in masks:
img = np.zeros(shape[0]*shape[1], dtype=np.uint8)
 s = mask.split()
for i in range(len(s)//2):
start = int(s[2*i])- 1
length = int(s[2*i+1])
 img[start:start+length] = 1
result.append(img.reshape(shape ).T)
 return result
learning_rate = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)
 batch = 64
epochs = 5
H = [0] * nets
for j in range(nets):
X_train2, X_val2, y_train2, y_val2 = train_test_split(X_train, y_train, test_size = 0.1)
 H[j] = model[j].fit_generator(datagen.flow(X_train2,y_train2, batch_size=batch),
epochs = epochs, validation_data =(X_val2,y_val2),
verbose = 1, steps_per_epoch=X_train2.shape[0] // batch
,callbacks=[learning_rate])
<save_to_csv>
class Score_eval() :
def __init__(self):
self.segmentation_df = pd.read_csv(SEGMENTATION ).set_index('ImageId')
 self.score, self.count = 0.0, 0
def put(self,pred,name):
true = get_mask_ind(name, self.segmentation_df)
 self.score += get_score(pred,true)
 self.count += 1
def evaluate(self):
return self.score/self.count
results = np.zeros(( test.shape[0],10))
for j in range(nets):
results = results + model[j].predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 sub = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 sub.to_csv("submission.csv",index=False )<import_modules>
def aug_unit(x,fwd=True,mask=False):
return x
def aug_flipV(x,fwd=True,mask=False):
return x.flip(2)if mask else x.flip(3)
 def aug_flipH(x,fwd=True,mask=False):
return x.flip(1)if mask else x.flip(2)
 def aug_T(x,fwd=True,mask=False):
return torch.transpose(x,1,2)if mask else torch.transpose(x,2,3)
 def aug_rot_2(x,fwd=True,mask=False):
return aug_flipV(aug_flipH(x,fwd,mask),fwd,mask)
 def aug_rot_4cr(x,fwd=True,mask=False):
return aug_flipV(aug_T(x,fwd,mask),fwd,mask)if fwd else \
aug_T(aug_flipV(x,fwd,mask),fwd,mask)
 def aug_rot_4cw(x,fwd=True,mask=False):
return aug_flipH(aug_T(x,fwd,mask),fwd,mask)if fwd else \
aug_T(aug_flipH(x,fwd,mask),fwd,mask)
 def aug_rot_2T(x,fwd=True,mask=False):
return aug_rot_2(aug_T(x,fwd,mask),fwd,mask)
 trms_side_on = [aug_unit,aug_flipH]
trms_top_down = [aug_unit,aug_flipV]
trms_dihedral = [aug_unit,aug_flipH,aug_flipV,aug_T,aug_rot_2,aug_rot_2T,
aug_rot_4cw,aug_rot_4cr]
<load_from_csv>
def enc_img(img):
return torch.transpose(torch.tensor(img),0,2 ).unsqueeze(0)
 def dec_img(img):
return to_np(torch.transpose(img.squeeze(0),0,2))
def display_augs(x,augs=aug_unit):
columns = 4
n = len(augs)
 rows = n//4 + 1
fig=plt.figure(figsize=(columns*4, rows*4))
img = enc_img(x)
 for i in range(rows):
for j in range(columns):
idx = j+i*columns
if idx >= n: break
fig.add_subplot(rows, columns, idx+1)
 plt.axis('off')
 plt.imshow(dec_img(augs[idx](img)))
 plt.show()
img = np.array(Image.open(os.path.join(TRAIN,'ce69faa4b.jpg')))
 display_augs(img,trms_dihedral )
train = pd.read_csv(".. /input/tmdb-box-office-prediction/train.csv")
 test = pd.read_csv(".. /input/tmdb-box-office-prediction/test.csv")
 imdb = pd.read_csv(".. /input/imdb-web-scraping-4/imdb.csv")
 train.head()<prepare_x_and_y>
def model_pred(learner, dl, F_save):
learner.model.eval() ;
name_list = dl.dataset.fnames
num_batchs = len(dl)
 t = tqdm(iter(dl), leave=False, total=num_batchs)
 count = 0
for x,y in t:
py = to_np(torch.sigmoid(learn.model(V(x))))
batch_size = len(py)
 for i in range(batch_size):
F_save(py[i],to_np(y[i]),name_list[count])
 count += 1
def pred_aug(x,aug=[aug_unit]):
pred = []
for aug_cur in aug:
py = to_np(aug_cur(torch.sigmoid(learn.model(V(aug_cur(x)))) ,
fwd=False, mask=True))
pred.append(py)
 pred = np.stack(pred, axis=0 ).mean(axis=0)
 return pred
def model_pred_aug(learner, dl, F_save, aug=[aug_unit]):
learner.model.eval() ;
name_list = dl.dataset.fnames
num_batchs = len(dl)
 t = tqdm(iter(dl), leave=False, total=num_batchs)
 count = 0
for x,y in t:
pred = pred_aug(x,aug)
 batch_size = len(pred)
 for i in range(batch_size):
F_save(pred[i],to_np(y[i]),name_list[count])
 count += 1
train_rows = train.shape[0]
y_train = np.log1p(train["revenue"] ).values
train_ID = train['id']
test_ID = test['id']
all_data = pd.concat([train, test], sort=False)
 all_data.drop(['revenue'], axis=1, inplace=True)
 all_data.drop(['id'], axis=1, inplace=True)
 all_data = pd.merge(all_data, imdb, how="left", on="imdb_id")
 all_data.head()<feature_engineering>
m = to_gpu(Unet34(get_base(False)))
 models = UnetModel(m )
for c in ['belongs_to_collection', 'genres', 'production_companies', 'production_countries', 'spoken_languages',
'Keywords', 'cast', 'crew']:
all_data[c] = all_data[c].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))<count_values>
sz = 768
bs = 8
md = get_data(sz,bs )
def print_few_values(col_name):
print("Sample values for", col_name)
 all_data[col_name].head(5 ).apply(lambda x: print(x))
def dictionary_sizes(col_name):
return(all_data[col_name].apply(lambda x: len(x)).value_counts())
 def print_dictionary_sizes(col_name):
print("
===================================================")
 print("Distribution of sizes for", col_name)
 print(dictionary_sizes(col_name))
def dict_name_list(d, name="name"):
return([i[name] for i in d] if d != {} else [])
 def dict_id_name_list(d, name="name"):
return([(i["id"],i[name])for i in d] if d != {} else [])
 def col_name_list(col_name, name="name"):
name_list_list = list(all_data[col_name].apply(lambda x: dict_name_list(x, name)).values)
 return([i for name_list in name_list_list for i in name_list])
 def col_id_name_list(col_name, name="name"):
tuple_list_list = list(all_data[col_name].apply(lambda x: dict_id_name_list(x, name)).values)
 return([i for tuple_list in tuple_list_list for i in tuple_list])
 def get_names_counter(col_name, name="name"):
name_list = col_name_list(col_name, name)
 return(collections.Counter(name_list))
def print_top_names(col_name, name="name"):
print("
===================================================")
 print("Top {0}s for {1}".format(name, col_name))
c = get_names_counter(col_name, name)
 print(c.most_common(20))
def EDA_dict(col_name):
print_few_values(col_name)
 print_dictionary_sizes(col_name)
 print_top_names(col_name)
 def add_dict_size_column(col_name):
all_data[col_name + "_size"] = all_data[col_name].apply(lambda x: len(x)if x != {} else 0)
 def add_dict_id_column(col_name):
c = col_name + "_id"
all_data[c] = all_data[col_name].apply(lambda x: x[0]["id"] if x != {} else 0)
 all_data[c] = all_data[c].astype("category")
 def add_dict_indicator_columns(col_name):
c = get_names_counter(col_name)
 top_names = [x[0] for x in c.most_common(20)]
for name in top_names:
all_data[col_name + "_" + name] = all_data[col_name].apply(lambda x: name in dict_name_list(x))
def drop_column(col_name):
all_data.drop([col_name], axis=1, inplace=True)
 def feature_engineer_dict(col_name):
add_dict_size_column(col_name)
 max_size = dictionary_sizes(col_name ).index.max()
if max_size == 1:
add_dict_id_column(col_name)
 else:
add_dict_indicator_columns(col_name)
 drop_column(col_name)
 def encode_column(col_name):
lbl = LabelEncoder()
lbl.fit(list(all_data[col_name].values))
all_data[col_name] = lbl.transform(list(all_data[col_name].values))<define_variables>
learn = ConvLearner(md, models)
 learn.models_path = PRETRAINED_SEGMENTATION_PATH
learn.load('Unet34_768_1')
 learn.models_path = PATH
col_name = "belongs_to_collection"
EDA_dict(col_name )<define_variables>
score = Score_eval()
process_pred = lambda yp, y, name : score.put(split_mask(yp),name)
 model_pred_aug(learn, md.val_dl, process_pred, trms_dihedral)
 print('
',score.evaluate() )
col_name="genres"
EDA_dict(col_name )<data_type_conversions>
ship_detection = pd.read_csv(DETECTION_TEST_PRED)
 ship_detection.head()
all_data.loc[all_data["release_date"].isnull() ]
all_data.loc[all_data["release_date"].isnull() , "release_date"] = "05/01/2000"
all_data["release_date"] = pd.to_datetime(all_data["release_date"])
 all_data["release_date_weekday"] = all_data["release_date"].dt.weekday.astype(int)
 all_data["release_date_month"] = all_data["release_date"].dt.month.astype(int)
 all_data["release_date_year"] = all_data["release_date"].dt.year.astype(int)
 all_data["release_date_year"] = np.where(all_data["release_date_year"]>2019, all_data["release_date_year"]-100, all_data["release_date_year"])
 all_data["imdb_year"]=all_data["imdb_year"].astype(int)
 print(all_data[abs(all_data["release_date_year"]-all_data["imdb_year"])>2][["imdb_id","release_date_year","imdb_year"]])
 all_data.loc[abs(all_data["release_date_year"]-all_data["imdb_year"])>2, "release_date_year"] = all_data["imdb_year"]
drop_column("release_date")
 drop_column("imdb_year" )<feature_engineering>
test_names = ship_detection.loc[ship_detection['p_ship'] > 0.5, ['id']]['id'].values.tolist()
test_names_nothing = ship_detection.loc[ship_detection['p_ship'] <= 0.5, ['id']]['id'].values.tolist()
len(test_names), len(test_names_nothing )
all_data.loc[all_data["runtime"].isnull() , "imdb_id"]
all_data.loc[all_data["runtime"]==0, "imdb_id"]
all_data.loc[all_data["runtime"]==0, "runtime"] = all_data["imdb_runtime"]
all_data.loc[all_data["runtime"].isnull() , "runtime"] = all_data["imdb_runtime"]
all_data["runtime"].fillna(all_data["runtime"].mean() , inplace=True)
 drop_column("imdb_runtime" )<categorify>
md = get_data(sz,bs)
 learn.set_data(md )
encode_column("original_language" )<define_variables>
def decode_mask(mask, shape=(768, 768)) :
pixels = mask.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )
def get_names() :
tuple_list_list = list(all_data["cast"].apply(lambda x: [(cm["name"].split(" ")[0],cm["gender"])for cm in x] ).values)
 tuple_list_list.extend(list(all_data["crew"].apply(lambda x: [(cm["name"].split(" ")[0],cm["gender"])for cm in x] ).values))
tuple_list = [i for tuple_list in tuple_list_list for i in tuple_list]
names = set()
c = [None] * 3
for gender in range(0, 3):
t = [i[0] for i in tuple_list if i[1] == gender]
names.update(set(t))
c[gender] = collections.Counter(t)
 names_df = pd.DataFrame(data = list(names), columns=["name"])
 names_df["female_count"] = names_df["name"].apply(lambda x: c[1][x])
 names_df["male_count"] = names_df["name"].apply(lambda x: c[2][x])
 names_df["unknown_count"] = names_df["name"].apply(lambda x: c[0][x])
 names_df["total_count"] = names_df.apply(lambda x: x["female_count"]+x["male_count"]+x["unknown_count"], axis=1)
 return(names_df)
 names_df = get_names()
def ClassifyName(row):
fcount = row["female_count"]
mcount = row["male_count"]
cl = "TBD"
gender = 0
if(fcount + mcount < 5):
cl = "too few names"
elif(fcount == 0):
gender = 2
cl = "high confidence"
elif(mcount == 0):
gender = 1
cl = "high confidence"
else:
if(mcount / float(fcount)< 0.1):
gender = 1
cl = "unisex - pick one"
elif(fcount / float(mcount)< 0.1):
gender = 2
cl = "unisex - pick one"
else:
cl = "unisex - TBD"
return [gender, cl]
names_df = names_df.merge(names_df.apply(lambda x: ClassifyName(x), axis=1, result_type="expand"), left_index=True, right_index=True)
 names_df.rename(columns={0:"gender", 1:"classification"}, inplace=True)
 names_to_gender = dict()
def update_names_to_gender(row):
if(row["gender"] > 0):
names_to_gender[row["name"]] = row["gender"]
j = names_df.apply(lambda x: update_names_to_gender(x), axis=1)
 def fix_unknown_gender_row(row):
for cm in row:
if cm["gender"] == 0:
name = cm["name"]
first_name = name.split(" ")[0]
if(first_name in names_to_gender):
cm["gender"] = names_to_gender[first_name]
def fix_unknown_gender_col(col_name):
all_data[col_name].apply(lambda x: fix_unknown_gender_row(x))<define_variables>
ship_list_dict = []
for name in test_names_nothing:
ship_list_dict.append({'ImageId':name,'EncodedPixels':np.nan} )
col_name = "crew"
EDA_dict(col_name)
 print_top_names(col_name, name="job" )<feature_engineering>
def enc_test(yp, y, name):
masks = split_mask(yp)
 if(len(masks)== 0):
ship_list_dict.append({'ImageId':name,'EncodedPixels':np.nan})
 for mask in masks:
ship_list_dict.append({'ImageId':name,'EncodedPixels':decode_mask(mask)} )
<define_variables><EOS> 
model_pred_aug(learn, md.test_dl, enc_test, trms_dihedral)
 pred_df = pd.DataFrame(ship_list_dict)
 pred_df.to_csv('submission.csv', index=False )