Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_pretrained>
warnings.filterwarnings("ignore" )
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_json_file(self, filename):
with open(filename, 'r', encoding='utf-8')as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns')
 file_sentences_sentiment_df = pd.DataFrame(
 {
'magnitude_sum': file_sentences_sentiment['magnitude'].sum(axis=0),
'score_sum': file_sentences_sentiment['score'].sum(axis=0),
'magnitude_mean': file_sentences_sentiment['magnitude'].mean(axis=0),
'score_mean': file_sentences_sentiment['score'].mean(axis=0),
'magnitude_var': file_sentences_sentiment['magnitude'].var(axis=0),
'score_var': file_sentences_sentiment['score'].var(axis=0),
}, index=[0] 
)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
df_sentiment = pd.concat([df_sentiment, file_sentences_sentiment_df], axis=1)
 df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = f'.. /input/petfinder-adoption-prediction/{mode}_sentiment/{pet_id}.json'
try:
sentiment_file = pet_parser.open_json_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob(f'.. /input/petfinder-adoption-prediction/{mode}_metadata/{pet_id}*.json'))
if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_json_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()<feature_engineering>
np.random.seed(2019)
 %matplotlib inline
print("done" )
aggregates = ['sum', 'mean', 'max', 'min', 'std']
sent_agg = ['sum']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 train_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 test_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()<concatenate>
def read_and_concat_dataset(training_path, test_path):
train = pd.read_csv(training_path)
 train['train'] = 1
test = pd.read_csv(test_path)
 test['train'] = 0
data = train.append(test, ignore_index=True)
 return train, test, data
train, test, data = read_and_concat_dataset('.. /input/train.csv', '.. /input/test.csv')
 data = data.set_index('PassengerId' )
all_sentiment_gr = pd.concat([train_sentiment_gr, test_sentiment_gr], ignore_index=True, sort=False)
 all_metadata_gr = pd.concat([train_metadata_gr, test_metadata_gr], ignore_index=True, sort=False)
 all_metadata_desc = pd.concat([train_metadata_desc, test_metadata_desc], ignore_index=True, sort=False)
 all_sentiment_desc = pd.concat([train_sentiment_desc, test_sentiment_desc], ignore_index=True, sort=False )<merge>
def counting_values(data, variable1, variable2):
return data[[variable1, variable2]][data[variable2].isnull() ==False].groupby([variable1], as_index=False ).mean().sort_values(by=variable2, ascending=False )
df_all = df_all.merge(
 all_sentiment_gr, how='left', on='PetID')
 df_all = df_all.merge(
 all_metadata_gr, how='left', on='PetID')
 df_all = df_all.merge(
 all_metadata_desc, how='left', on='PetID')
 df_all = df_all.merge(
 all_sentiment_desc, how='left', on='PetID')
 all_columns.extend(list(all_sentiment_gr.columns[1:])+list(all_metadata_gr.columns[1:]))<feature_engineering>
counting_values(data, 'Sex','Survived' )
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
X_text = df_all[text_columns].copy()
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('none' )<feature_engineering>
data['Women'] = np.where(data.Sex=='female',1,0)
 comparing(data, 'Women','Survived' )
n_components = 16
text_features = []
for i in X_text.columns:
print(f'generating features from: {i}')
 tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_{}_'.format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1 )<concatenate>
data.isnull().sum()
df_all = pd.concat([df_all, text_features.reset_index(drop=True)], axis=1)
 all_columns.extend(text_features.columns )<feature_engineering>
data.isnull().sum()
train_image_nums = Counter([i.split('/')[-1].split('-')[0] for i in train_image_files])
 test_image_nums = Counter([i.split('/')[-1].split('-')[0] for i in test_image_files])
 train['image_num'] = train.PetID.apply(lambda x: train_image_nums[x] if x in train_image_nums else 0)
 test['image_num'] = test.PetID.apply(lambda x: test_image_nums[x] if x in test_image_nums else 0 )<define_variables>
data.groupby('Pclass' ).Fare.mean()
train_df_ids = train[['PetID']]
test_df_ids = test[['PetID']]
train_df_imgs = pd.DataFrame(train_image_files)
 train_df_imgs.columns = ['image_filename']
train_imgs_pets = train_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 def getSize(filename):
st = os.stat(filename)
 return st.st_size
def getDimensions(filename):
img_size = Image.open(filename ).size
return img_size
train_df_imgs['image_size'] = train_df_imgs['image_filename'].apply(getSize)
 train_df_imgs['temp_size'] = train_df_imgs['image_filename'].apply(getDimensions)
 train_df_imgs['width'] = train_df_imgs['temp_size'].apply(lambda x : x[0])
 train_df_imgs['height'] = train_df_imgs['temp_size'].apply(lambda x : x[1])
 train_df_imgs = train_df_imgs.drop(['temp_size'], axis=1)
 test_df_imgs['image_size'] = test_df_imgs['image_filename'].apply(getSize)
 test_df_imgs['temp_size'] = test_df_imgs['image_filename'].apply(getDimensions)
 test_df_imgs['width'] = test_df_imgs['temp_size'].apply(lambda x : x[0])
 test_df_imgs['height'] = test_df_imgs['temp_size'].apply(lambda x : x[1])
 test_df_imgs = test_df_imgs.drop(['temp_size'], axis=1)
 aggs = {
'image_size': ['sum', 'mean', 'max', 'min', 'std'],
'width': ['sum', 'mean', 'max', 'min', 'std'],
'height': ['sum', 'mean', 'max', 'min', 'std'],
}
agg_train_imgs = train_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()
agg_test_imgs = test_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index()
agg_imgs = pd.concat([agg_train_imgs, agg_test_imgs], axis=0 ).reset_index(drop=True )<merge>
data.Fare = data.Fare.fillna(0 )
df_all = pd.merge(df_all,agg_imgs,on='PetID',how='left')
 all_columns.extend(new_columns + ['image_num'] )<feature_engineering>
print(data.Embarked.value_counts())
 data.Embarked = data.Embarked.fillna('S' )
breed_job = df_all[['Breed1','Breed2']]
breed_job['mixed_breed'] =(( breed_job[['Breed1','Breed2']]==307 ).sum(axis=1)>0 ).astype('int')
 breed_job.loc[breed_job['Breed1']==307,'Breed1'] = 0
breed_job.loc[breed_job['Breed2']==307,'Breed2'] = 0
breed_job.loc[breed_job['mixed_breed']==1,'Breed1'] = breed_job.loc[breed_job['mixed_breed']==1,'Breed1']+breed_job.loc[breed_job['mixed_breed']==1,'Breed2']
breed_job.loc[breed_job['mixed_breed']==1,'Breed2'] = 0
breed_job.loc[(breed_job['mixed_breed']==0)&(breed_job['Breed2']!=0),'extra_mixed'] = 1
breed_job.loc[breed_job['extra_mixed']==1,'Breed2'] = 0
breed_job['mixed_breed'] = breed_job['mixed_breed'] + breed_job['extra_mixed'].fillna(0)
 df_all = df_all.drop(['Breed1', 'Breed2',],axis=1)
 df_all = pd.concat(
 [df_all, breed_job[['Breed1','mixed_breed']]], axis=1 )<categorify>
data.Cabin = data.Cabin.fillna('Unknown_Cabin')
 data['Cabin'] = data['Cabin'].str[0]
df_all['Type'] = df_all['Type'].map({1:1,2:0})
 df_all = df_all.rename(columns={'Type':'is_dog'} )<merge>
data.groupby('Pclass' ).Cabin.value_counts()
df_all_breed_names = df_all[['Breed1']].merge(
 labels_breed[['BreedID','BreedName']], how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))<categorify>
data['Cabin'] = np.where(( data.Pclass==1)&(data.Cabin=='U'),'C',
np.where(( data.Pclass==2)&(data.Cabin=='U'),'D',
np.where(( data.Pclass==3)&(data.Cabin=='U'),'G',
np.where(data.Cabin=='T','C',data.Cabin))))
def path_to_tensor(img_path):
img = image.load_img(img_path, target_size=(299, 299))
x = image.img_to_array(img)
 return np.expand_dims(x, axis=0)
 def paths_to_tensor(img_paths):
list_of_tensors = [path_to_tensor(img_path)for img_path in img_paths]
return np.vstack(list_of_tensors )<feature_engineering>
data['Title'] = data.Name.str.extract('([A-Za-z]+)\.', expand=False)
 pd.crosstab(data['Title'], data['Sex'])
 data = data.drop('Name',axis=1 )
predicted_df = df_all[(df_all.Breed1 == 0)&(df_all.mixed_breed == 1)& df_all.is_dog][['PetID', 'is_train']].copy().reset_index(drop=True)
 predicted_df['img_path'] = predicted_df.apply(lambda x: f".. /input/petfinder-adoption-prediction/{'train' if x['is_train'] else 'test'}_images/{x['PetID']}-1.jpg", axis=1)
 predicted_df = predicted_df[predicted_df.img_path.map(os.path.exists)].reset_index(drop=True)
 predicted_df['pred_breed'] = 0
fns = predicted_df.img_path.values<categorify>
data['Title'] = np.where(( data.Title=='Capt')|(data.Title=='Countess')|(data.Title=='Don')|(data.Title=='Dona')
 |(data.Title=='Jonkheer')|(data.Title=='Lady')|(data.Title=='Sir')|(data.Title=='Major')|(data.Title=='Rev')|(data.Title=='Col'),'Other',data.Title)
 data['Title'] = data['Title'].replace('Ms','Miss')
 data['Title'] = data['Title'].replace('Mlle','Miss')
 data['Title'] = data['Title'].replace('Mme','Mrs' )
ImageFile.LOAD_TRUNCATED_IMAGES = True
def create_generator(batch_size):
while True:
for i in range(0, len(fns), batch_size):
chunk = fns[i:i+batch_size]
x = paths_to_tensor(chunk ).astype('float32')/255
yield x<define_variables>
data.groupby('Title' ).Age.mean()
batchsize = 256
pred_generator = create_generator(batchsize )<choose_model_class>
data['Age'] = np.where(( data.Age.isnull())&(data.Title=='Master'),5,
np.where(( data.Age.isnull())&(data.Title=='Miss'),22,
np.where(( data.Age.isnull())&(data.Title=='Mr'),32,
np.where(( data.Age.isnull())&(data.Title=='Mrs'),37,
np.where(( data.Age.isnull())&(data.Title=='Other'),45,
np.where(( data.Age.isnull())&(data.Title=='Dr'),44,data.Age))))))
inp_ten = Input(shape=(299, 299, 3))
x = InceptionResNetV2(
 weights='.. /input/inceptionresnetv2/inception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.h5',
include_top=False,
pooling=None
)(inp_ten)
 x = GlobalAveragePooling2D()(x)
 out = Dense(133, activation='softmax' )(x)
 model = Model(inp_ten, out)
 model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.summary()
model.load_weights('.. /input/dog-identification-pretrained/dog_identification_model.hdf5' )<categorify>
data['FamilySize'] = data.SibSp + data.Parch + 1
data['Mother'] = np.where(( data.Title=='Mrs')&(data.Parch >0),1,0)
 data['Free'] = np.where(data['Fare']==0, 1,0)
 data = data.drop(['SibSp','Parch','Sex'],axis=1 )
breed_dict = {i:j
for i, j in zip(
 labels_breed[labels_breed.Type == 1].BreedName.apply(lambda x: x.lower().replace(' ', '_')).values,
labels_breed[labels_breed.Type == 1].BreedID.values)}<define_variables>
TypeOfTicket = []
for i in range(len(data.Ticket)) :
ticket = data.Ticket.iloc[i]
for c in string.punctuation:
ticket = ticket.replace(c,"")
 splited_ticket = ticket.split(" ")
 if len(splited_ticket)== 1:
TypeOfTicket.append('NO')
 else:
TypeOfTicket.append(splited_ticket[0])
 data['TypeOfTicket'] = TypeOfTicket
data.TypeOfTicket.value_counts()
data['TypeOfTicket'] = np.where(( data.TypeOfTicket!='NO')&(data.TypeOfTicket!='PC')&(data.TypeOfTicket!='CA')& 
(data.TypeOfTicket!='A5')&(data.TypeOfTicket!='SOTONOQ'),'other',data.TypeOfTicket)
 data = data.drop('Ticket',axis=1 )
dog_names = ['in/001.Affenpinscher', 'in/002.Afghan_hound', 'in/003.Airedale_terrier', 'in/004.Akita', 'in/005.Alaskan_malamute', 'in/006.American_eskimo_dog', 'in/007.American_foxhound', 'in/008.American_staffordshire_terrier', 'in/009.American_water_spaniel', 'in/010.Anatolian_shepherd_dog', 'in/011.Australian_cattle_dog', 'in/012.Australian_shepherd', 'in/013.Australian_terrier', 'in/014.Basenji', 'in/015.Basset_hound', 'in/016.Beagle', 'in/017.Bearded_collie', 'in/018.Beauceron', 'in/019.Bedlington_terrier', 'in/020.Belgian_malinois', 'in/021.Belgian_sheepdog', 'in/022.Belgian_tervuren', 'in/023.Bernese_mountain_dog', 'in/024.Bichon_frise', 'in/025.Black_and_tan_coonhound', 'in/026.Black_russian_terrier', 'in/027.Bloodhound', 'in/028.Bluetick_coonhound', 'in/029.Border_collie', 'in/030.Border_terrier', 'in/031.Borzoi', 'in/032.Boston_terrier', 'in/033.Bouvier_des_flandres', 'in/034.Boxer', 'in/035.Boykin_spaniel', 'in/036.Briard', 'in/037.Brittany', 'in/038.Brussels_griffon', 'in/039.Bull_terrier', 'in/040.Bulldog', 'in/041.Bullmastiff', 'in/042.Cairn_terrier', 'in/043.Canaan_dog', 'in/044.Cane_corso', 'in/045.Cardigan_welsh_corgi', 'in/046.Cavalier_king_charles_spaniel', 'in/047.Chesapeake_bay_retriever', 'in/048.Chihuahua', 'in/049.Chinese_crested', 'in/050.Chinese_shar-pei', 'in/051.Chow_chow', 'in/052.Clumber_spaniel', 'in/053.Cocker_spaniel', 'in/054.Collie', 'in/055.Curly-coated_retriever', 'in/056.Dachshund', 'in/057.Dalmatian', 'in/058.Dandie_dinmont_terrier', 'in/059.Doberman_pinscher', 'in/060.Dogue_de_bordeaux', 'in/061.English_cocker_spaniel', 'in/062.English_setter', 'in/063.English_springer_spaniel', 'in/064.English_toy_spaniel', 'in/065.Entlebucher_mountain_dog', 'in/066.Field_spaniel', 'in/067.Finnish_spitz', 'in/068.Flat-coated_retriever', 'in/069.French_bulldog', 'in/070.German_pinscher', 'in/071.German_shepherd_dog', 'in/072.German_shorthaired_pointer', 'in/073.German_wirehaired_pointer', 'in/074.Giant_schnauzer', 'in/075.Glen_of_imaal_terrier', 'in/076.Golden_retriever', 'in/077.Gordon_setter', 'in/078.Great_dane', 'in/079.Great_pyrenees', 'in/080.Greater_swiss_mountain_dog', 'in/081.Greyhound', 'in/082.Havanese', 'in/083.Ibizan_hound', 'in/084.Icelandic_sheepdog', 'in/085.Irish_red_and_white_setter', 'in/086.Irish_setter', 'in/087.Irish_terrier', 'in/088.Irish_water_spaniel', 'in/089.Irish_wolfhound', 'in/090.Italian_greyhound', 'in/091.Japanese_chin', 'in/092.Keeshond', 'in/093.Kerry_blue_terrier', 'in/094.Komondor', 'in/095.Kuvasz', 'in/096.Labrador_retriever', 'in/097.Lakeland_terrier', 'in/098.Leonberger', 'in/099.Lhasa_apso', 'in/100.Lowchen', 'in/101.Maltese', 'in/102.Manchester_terrier', 'in/103.Mastiff', 'in/104.Miniature_schnauzer', 'in/105.Neapolitan_mastiff', 'in/106.Newfoundland', 'in/107.Norfolk_terrier', 'in/108.Norwegian_buhund', 'in/109.Norwegian_elkhound', 'in/110.Norwegian_lundehund', 'in/111.Norwich_terrier', 'in/112.Nova_scotia_duck_tolling_retriever', 'in/113.Old_english_sheepdog', 'in/114.Otterhound', 'in/115.Papillon', 'in/116.Parson_russell_terrier', 'in/117.Pekingese', 'in/118.Pembroke_welsh_corgi', 'in/119.Petit_basset_griffon_vendeen', 'in/120.Pharaoh_hound', 'in/121.Plott', 'in/122.Pointer', 'in/123.Pomeranian', 'in/124.Poodle', 'in/125.Portuguese_water_dog', 'in/126.Saint_bernard', 'in/127.Silky_terrier', 'in/128.Smooth_fox_terrier', 'in/129.Tibetan_mastiff', 'in/130.Welsh_springer_spaniel', 'in/131.Wirehaired_pointing_griffon', 'in/132.Xoloitzcuintli', 'in/133.Yorkshire_terrier']<predict_on_test>
counting_values(data, 'Title','Survived' )
preds = model.predict_generator(pred_generator, steps=len(fns)// batchsize + 1, verbose=1)
 preds = np.argmax(preds, axis=1)
 mapping = {
'american_foxhound': 'foxhound',
'anatolian_shepherd_dog': 'anatolian_shepherd',
'australian_cattle_dog': 'australian_cattle_dog/blue_heeler',
'belgian_malinois': 'belgian_shepherd_malinois',
'belgian_sheepdog': 'belgian_shepherd_dog_sheepdog',
'belgian_tervuren': 'belgian_shepherd_tervuren',
'bouvier_des_flandres': 'bouvier_des_flanders',
'brittany': 'brittany_spaniel',
'bulldog': 'american_bulldog',
'cane_corso': 'cane_corso_mastiff',
'cardigan_welsh_corgi': 'welsh_corgi',
'chinese_crested': 'chinese_crested_dog',
'chinese_shar-pei': 'shar_pei',
'dandie_dinmont_terrier': 'dandi_dinmont_terrier',
'entlebucher_mountain_dog': 'entlebucher',
'icelandic_sheepdog': 'shetland_sheepdog_sheltie',
'irish_red_and_white_setter': 'irish_setter',
'miniature_schnauzer': 'schnauzer',
'newfoundland': 'newfoundland_dog',
'nova_scotia_duck_tolling_retriever': 'nova_scotia_duck-tolling_retriever',
'parson_russell_terrier': 'jack_russell_terrier',
'pembroke_welsh_corgi': 'welsh_corgi',
'plott': 'plott_hound',
'wirehaired_pointing_griffon': 'german_wirehaired_pointer',
'xoloitzcuintli': 'xoloitzcuintle/mexican_hairless',
'yorkshire_terrier': 'yorkshire_terrier_yorkie'
}
def get_breed_id(prediction):
dog_name = dog_names[prediction].split('.')[-1].lower()
dog_name = dog_name if dog_name not in mapping else mapping[dog_name]
return breed_dict[dog_name]
predicted_df['pred_breed'] = [get_breed_id(x)for x in preds]
del model; gc.collect()
K.clear_session()<merge>
counting_values(data, 'TypeOfTicket','Survived' )
predicted_df = predicted_df[['PetID', 'pred_breed']]
df_all = df_all.merge(predicted_df, on='PetID', how='left')
 del predicted_df; gc.collect()
df_all.pred_breed = df_all.pred_breed.fillna(0)
 get_values = lambda i, j: int(j)if j != 0 else i
df_all.pred_breed = [get_values(i, j)for i,j in zip(df_all.Breed1.values, df_all.pred_breed.values)]<data_type_conversions>
counting_values(data, 'Cabin','Survived' )
for e in ['hair','domestic', 'short', 'medium', 'retriever', 'terrier', 'tabby', 'long']:
df_all_breed_names['Breed_{}'.format(e)] =(df_all_breed_names.BreedName.apply(lambda x: e in str(x ).lower())).astype(int)
 breed_detail_features = ['Breed_{}'.format(e)for e in ['hair','domestic', 'short', 'medium', 'retriever', 'terrier', 'tabby', 'long']]
df_all = pd.concat(
 [df_all, df_all_breed_names[breed_detail_features]], axis=1 )<categorify>
data = pd.get_dummies(data )
le = LabelEncoder()
df_all['Breed1_le'] = le.fit_transform(df_all['Breed1'] )<concatenate>
trainX, testX, trainY, testY = train_test_split(data[data.Survived.isnull() ==False].drop('Survived',axis=1),data.Survived[data.Survived.isnull() ==False],test_size=0.30, random_state=2019 )
all_columns = []
all_columns.extend(breed_columns )<categorify>
Results = pd.DataFrame({'Model': [],'Accuracy Score': [], 'Recall':[], 'F1score':[]} )
df_all['Age_bins'] = pd.cut(np.log1p(df_all.Age),9,labels=['Age_{}'.format(e)for e in range(9)])
 le = LabelEncoder()
df_all['Age_bins'] = le.fit_transform(df_all['Age_bins'])
 age_columns = ['Age_bins']<concatenate>
from sklearn.metrics import f1_score
from sklearn.metrics import recall_score
from sklearn.metrics import accuracy_score
all_columns.extend(age_columns )<feature_engineering>
model = DecisionTreeClassifier(max_depth=4)
 model.fit(trainX, trainY)
 y_pred = model.predict(testX)
 res = pd.DataFrame({"Model":['DecisionTreeClassifier'],
"Accuracy Score": [accuracy_score(y_pred,testY)],
"Recall": [recall_score(testY, y_pred)],
"F1score": [f1_score(testY, y_pred)]})
 Results = Results.append(res )
df_all['Name_isna'] = df_all['Name'].isna().astype(int)
 df_all.loc[~df_all['Name'].isna() ,'Name_len'] = df_all.loc[~df_all['Name'].isna() ,'Name'].apply(lambda x : len(str(x ).split()))
df_all['Name_len'] = df_all['Name_len'].fillna(0)
 name_with_numbers = [e for e in df_all.Name.unique() if len(set([str(t)for t in range(10)] ).intersection(set(str(e ).split())))>0]
df_all['Name_with_numbers'] =(df_all.Name.isin(name_with_numbers)).astype(int )<define_variables>
pd.crosstab(testY, y_pred, rownames=['Real data'], colnames=['Predicted'] )
name_columns = ['Name_len','Name_isna',"Name_with_numbers"]<concatenate>
model = RandomForestClassifier(n_estimators=2500, max_depth=4)
 model.fit(trainX, trainY)
 y_pred = model.predict(testX)
 res = pd.DataFrame({"Model":['RandomForestClassifier'],
"Accuracy Score": [accuracy_score(y_pred,testY)],
"Recall": [recall_score(testY, y_pred)],
"F1score": [f1_score(testY, y_pred)]})
 Results = Results.append(res )
all_columns.extend(name_columns )<feature_engineering>
pd.crosstab(testY, y_pred, rownames=['Real data'], colnames=['Predicted'] )
df_all['Color_range'] =(df_all[['Color1','Color2','Color3',]]>0 ).sum(axis=1 )<categorify>
model = KNeighborsClassifier()
model.fit(trainX, trainY)
 y_pred = model.predict(testX)
 res = pd.DataFrame({"Model":['KNeighborsClassifier'],
"Accuracy Score": [accuracy_score(y_pred,testY)],
"Recall": [recall_score(testY, y_pred)],
"F1score": [f1_score(testY, y_pred)]})
 Results = Results.append(res )
df_all['Fee_bins']= pd.cut(np.log1p(df_all.Fee),5,labels=['Fee_{}'.format(e)for e in range(5)])
 le = LabelEncoder()
df_all['Fee_bins'] = le.fit_transform(df_all['Fee_bins'])
<define_variables>
pd.crosstab(testY, y_pred, rownames=['Real data'], colnames=['Predicted'] )
fee_columns = ['Fee_bins']<concatenate>
model = SVC()
model.fit(trainX, trainY)
 y_pred = model.predict(testX)
 res = pd.DataFrame({"Model":['SVC'],
"Accuracy Score": [accuracy_score(y_pred,testY)],
"Recall": [recall_score(testY, y_pred)],
"F1score": [f1_score(testY, y_pred)]})
 Results = Results.append(res )
all_columns.extend(fee_columns )<categorify>
pd.crosstab(testY, y_pred, rownames=['Real data'], colnames=['Predicted'] )
df_all["state_gdp"] = df_all.State.map(state_gdp)
 df_all["state_population"] = df_all.State.map(state_population)
 le = LabelEncoder()
df_all['State_le'] = le.fit_transform(df_all['State'] )<define_variables>
model = LogisticRegression()
model.fit(trainX, trainY)
 y_pred = model.predict(testX)
 res = pd.DataFrame({"Model":['LogisticRegression'],
"Accuracy Score": [accuracy_score(y_pred,testY)],
"Recall": [recall_score(testY, y_pred)],
"F1score": [f1_score(testY, y_pred)]})
 Results = Results.append(res )
state_columns = ['state_gdp',
'state_population',
'State_le',]<concatenate>
pd.crosstab(testY, y_pred, rownames=['Real data'], colnames=['Predicted'] )
all_columns.extend(state_columns )<categorify>
model = XGBClassifier(learning_rate=0.001,n_estimators=2500,
max_depth=4, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006)
 model.fit(trainX, trainY)
 y_pred = model.predict(testX)
 res = pd.DataFrame({"Model":['XGBClassifier'],
"Accuracy Score": [accuracy_score(y_pred,testY)],
"Recall": [recall_score(testY, y_pred)],
"F1score": [f1_score(testY, y_pred)]})
 Results = Results.append(res )
df_all['is_VideoAmt'] =(df_all.VideoAmt>0 ).astype('int')
 df_all['PhotoAmt_bins']= pd.cut(np.log1p(df_all.PhotoAmt),5,labels=['PhotoAmt_{}'.format(e)for e in range(5)])
 le = LabelEncoder()
df_all['PhotoAmt_bins'] = le.fit_transform(df_all['PhotoAmt_bins'] )<define_variables>
pd.crosstab(testY, y_pred, rownames=['Real data'], colnames=['Predicted'] )
<concatenate><EOS> 
trainX = data[data.Survived.isnull() ==False].drop(['Survived','train'],axis=1)
 trainY = data.Survived[data.Survived.isnull() ==False]
testX = data[data.Survived.isnull() ==True].drop(['Survived','train'],axis=1)
 model = XGBClassifier(learning_rate=0.001,n_estimators=2500,
max_depth=4, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006)
 model.fit(trainX, trainY)
 test = data[data.train==0]
test['Survived'] = model.predict(testX ).astype(int)
 test = test.reset_index()
test[['PassengerId','Survived']].to_csv("submissionXGB.csv",index=False)
 print("done1" )