Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<merge>
warnings.filterwarnings('ignore')
 %config InlineBackend.figure_format = 'retina'
PATH = '.. /input/titanic/'
list(os.listdir(PATH))
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how="left", left_on="PetID", right_on="pet_id")
 train_proc = train_proc.merge(
 train_meta_gr, how="left", left_on="PetID", right_on="pet_id")
 train_proc = train_proc.merge(
 train_sentiment_desc, how="left", left_on="PetID", right_on="pet_id")
 train_proc = train_proc.merge(
 train_meta_desc, how="left", left_on="PetID", right_on = "pet_id")
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how="left", left_on="PetID", right_on="pet_id")
 test_proc = test_proc.merge(
 test_meta_gr, how="left", left_on="PetID", right_on="pet_id")
 test_proc = test_proc.merge(
 test_sentiment_desc, how="left", left_on="PetID", right_on="pet_id")
 test_proc = test_proc.merge(
 test_meta_desc, how="left", left_on="PetID", right_on = "pet_id" )<drop_column>
train = pd.read_csv(PATH+'train.csv')
 test = pd.read_csv(PATH+'test.csv')
 submission = pd.read_csv(PATH+'gender_submission.csv' )
train_proc.drop(train_proc.filter(
 regex="pet_id", axis=1 ).columns.tolist() ,
axis=1,
inplace=True)
 test_proc.drop(test_proc.filter(
 regex="pet_id", axis=1 ).columns.tolist() ,
axis=1,
inplace=True)
 train_proc.head()<categorify>
train = train.drop(columns=['PassengerId', 'Ticket'])
 test = test.drop(columns=['PassengerId', 'Ticket']);
train_proc.language.fillna("", inplace=True)
 test_proc.language.fillna("", inplace=True)
 langs = train_proc.language.unique()
encode_dict = {k: i for i, k in enumerate(langs)}
train_proc.language = train_proc.language.map(encode_dict)
 test_proc.language = test_proc.language.map(encode_dict )<concatenate>
train['Deck'] = train['Cabin'].astype(str ).str[0]
train['Deck'] = train['Deck'].map({'A': 1, 'B': 1, 'C': 1, 'D': 1, 'E':1,
'F':1, 'G': 1, 'T': 0, 'n' : 0} ).astype(int)
 test['Deck'] = test['Cabin'].astype(str ).str[0]
test['Deck'] = test['Deck'].map({'A': 1, 'B': 1, 'C': 1, 'D': 1, 'E':1,
'F':1, 'G': 1, 'T': 0, 'n' : 0} ).astype(int)
 train = train.drop(columns=['Cabin'])
 test = test.drop(columns=['Cabin']);
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False)
 X_temp = X.copy()
text_columns = [
"Description",
"entity",
"desc"]
categorical_columns = [
"Type", "Breed1", "Breed2", "Gender",
"Color1", "Color2", "Color3", "MaturitySize",
"FurLength", "Vaccinated", "Dewormed", "Sterilized",
"State", "language"
]
drop_columns = [
"PetID", "Name", "RescuerID"
]<merge>
train = train.dropna(subset=['Embarked'] ).reset_index(drop=True )
rescuer_count = X.groupby(["RescuerID"])["PetID"].count().reset_index()
rescuer_count.columns = ["RescuerID", "RescuerID_COUNT"]
X_temp = X_temp.merge(rescuer_count, how="left", on="RescuerID" )<feature_engineering>
missing = test.isnull().sum()
missing = missing[missing > 0]
missing
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna("none" )<categorify>
test['Fare'] = test['Fare'].fillna(test['Fare'].mode() [0] )
X_temp["len_description"] = X_text["Description"].map(len)
 X_temp["len_meta_desc"] = X_text["desc"].map(len)
 X_temp["len_entity"] = X_text["entity"].map(len )<feature_engineering>
train['Age'] = train['Age'].fillna(train['Age'].median())
 test['Age'] = test['Age'].fillna(test['Age'].median() )
n_components = 16
text_features = []
for i in X_text.columns:
print(f"generating features from: {i}")
 tfv = TfidfVectorizer(
 min_df=2,
strip_accents="unicode",
analyzer="word",
token_pattern=r"(?u)\b\w+\b",
ngram_range=(1, 3),
use_idf=1,
smooth_idf=1,
sublinear_tf=1)
 svd = TruncatedSVD(
 n_components=n_components,
random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix("Tfidf_{}_".format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp.drop(i, axis=1, inplace=True )<define_variables>
train['Family_Name'] = train['Name'].apply(lambda x : x.split(',')[0])
 train['First_Name'] = train['Name'].apply(lambda x : x.split(',')[1])
 train['Honorific'] = train['First_Name'].apply(lambda x : x.split('.')[0])
 train = train.drop(columns=['Name', 'First_Name'])
 test['Family_Name'] = test['Name'].apply(lambda x : x.split(',')[0])
 test['First_Name'] = test['Name'].apply(lambda x : x.split(',')[1])
 test['Honorific'] = test['First_Name'].apply(lambda x : x.split('.')[0])
 test = test.drop(columns=['Name', 'First_Name'] )
train_image_files = sorted(
 glob.glob(".. /input/petfinder-adoption-prediction/train_images/*.jpg"))
test_image_files = sorted(
 glob.glob(".. /input/petfinder-adoption-prediction/test_images/*.jpg"))
train_df_imgs = pd.DataFrame(train_image_files)
 test_df_imgs = pd.DataFrame(test_image_files)
 train_df_imgs.columns = ["image_file_name"]
test_df_imgs.columns = ["image_file_name"]
train_imgs_pets = train_df_imgs["image_file_name"].apply(
 lambda x: x.split("/")[-1].split("-")[0])
 test_imgs_pets = test_df_imgs["image_file_name"].apply(
 lambda x: x.split("/")[-1].split("-")[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 def get_size(filename):
st = os.stat(filename)
 return st.st_size
def get_dimensions(filename):
img_size = Image.open(filename ).size
return img_size
train_df_imgs["image_size"] = train_df_imgs["image_file_name"].apply(get_size)
 test_df_imgs["image_size"] = test_df_imgs["image_file_name"].apply(get_size)
 train_df_imgs["temp_size"] = train_df_imgs["image_file_name"].apply(get_dimensions)
 test_df_imgs["temp_size"] = test_df_imgs["image_file_name"].apply(get_dimensions)
 train_df_imgs["width"] = train_df_imgs["temp_size"].apply(lambda x: x[0])
 test_df_imgs["width"] = test_df_imgs["temp_size"].apply(lambda x: x[0])
 train_df_imgs["height"] = train_df_imgs["temp_size"].apply(lambda x: x[1])
 test_df_imgs["height"] = test_df_imgs["temp_size"].apply(lambda x: x[1])
 train_df_imgs.drop(["temp_size"], axis=1, inplace=True)
 test_df_imgs.drop(["temp_size"], axis=1, inplace=True)
 aggs = {
"image_size": ["sum", "mean", "var"],
"width": ["sum", "mean", "var"],
"height": ["sum", "mean", "var"]
}
agg_train_imgs = train_df_imgs.groupby("PetID" ).agg(aggs)
 new_columns = [
k + "_" + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs = test_df_imgs.groupby("PetID" ).agg(aggs)
 agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index() 
<merge>
sc = StandardScaler()
train[['Age', 'Fare']] = sc.fit_transform(train[['Age','Fare']])
 test[['Age', 'Fare']] = sc.fit_transform(test[['Age','Fare']] )
agg_imgs = pd.concat([agg_train_imgs, agg_test_imgs], axis=0 ).reset_index(drop=True)
 X_temp = X_temp.merge(agg_imgs, how="left", on="PetID")
 X_temp.head()<drop_column>
display('Train Dataset')
 display(train.head(3))
display('Test Dataset')
 display(test.head(3))
X_temp.drop(drop_columns, axis=1, inplace=True)
 X_temp.head()<feature_engineering>
h2o.init()
train_frame = h2o.H2OFrame(train)
 test_frame = h2o.H2OFrame(test)
 train_frame['Survived'] = train_frame['Survived'].asfactor()
x = train_frame.columns
y = 'Survived'
x.remove(y )
for c in new_columns:
X_temp.loc[:, c] = X_temp[c].map(lambda x: np.log1p(x))
X_temp.head()<drop_column>
%%time
model = H2OAutoML(max_models=10, seed=42, nfolds=5)
 model.train(x=x, y=y, training_frame=train_frame)
 lb = model.leaderboard
lb
X_train = X_temp.loc[np.isfinite(X_temp.AdoptionSpeed), :]
X_test = X_temp.loc[~np.isfinite(X_temp.AdoptionSpeed), :]
X_test.drop(["AdoptionSpeed"], axis=1, inplace=True)
 assert X_train.shape[0] == train.shape[0]
assert X_test.shape[0] == test.shape[0]<count_missing_values>
m = h2o.get_model(lb[2, "model_id"])
 m.varimp_plot()
X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1)
 X_train_non_null.isnull().any().any() , X_test_non_null.isnull().any().any()<feature_engineering>
train_frame_b = train_frame[:, ['Family_Name', 'Honorific', 'Sex', 'Pclass', 'Fare', 'Deck', 'Survived']]
train_frame_b['Survived'] = train_frame_b['Survived'].asfactor()
x_b = train_frame_b.columns
y_b = 'Survived'
x_b.remove(y_b)
 model_b = H2OAutoML(max_models=10, seed=42, nfolds=5)
 model_b.train(x=x_b, y=y_b, training_frame=train_frame_b)
 lb_b = model_b.leaderboard
lb_b
<prepare_x_and_y><EOS> 
predictions = model.leader.predict(test_frame);
predictions = predictions.as_data_frame()
predictions = predictions.predict
submission_frame = h2o.H2OFrame(submission)
 passenger_id = submission_frame['PassengerId'].as_data_frame()
submission_final = pd.concat([passenger_id, predictions], axis=1, ignore_index=False)
 submission_final.columns = ['PassengerId', 'Survived']
submission_final.to_csv('submission.csv', index=False)
 print('Submission file saved!' )