Kaggle competition: Titanic - Machine Learning from Disaster
del glove_word_embed
gc.collect()<feature_engineering>
train_data=pd.read_csv('/kaggle/input/titanic/train.csv')
 test_data=pd.read_csv('/kaggle/input/titanic/test.csv' )
n_components = 16
text_features = []
for i in X_text.columns:
print(f'generating features from: {i}')
 tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_{}_'.format(i))
text_features.append(svd_col)
 tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='char',
ngram_range=(2, 5), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_char_{}_'.format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )<merge>
train_data.isnull().sum()
X_temp = X_temp.merge(img_features, how='left', on='PetID' )<merge>
test_data.isnull().sum()
X_img_row_temp = X_temp[['PetID','AdoptionSpeed']].merge(img_row, how='left', on='PetID' )<drop_column>
train_data['Age'] = train_data['Age'].fillna(30 )
X_img_row_temp = X_img_row_temp.drop(['PetID'], axis = 1)
 X_text_image = X_img_row_temp[len_train:].drop(['AdoptionSpeed'], axis = 1 )<feature_engineering>
train_data = train_data.drop(['Cabin'],axis=1)
 test_data = test_data.drop(['Cabin'],axis=1 )
img_train, img_test = run_xgb(xgb_params,X_img_row_temp[:len_train],X_text_image)
 X_temp['img_pred'] = np.concatenate([img_train,img_test.mean(axis = 1)] )<define_variables>
train_data['Embarked'].value_counts()
train_df_ids = train[['PetID']]
test_df_ids = test[['PetID']]
train_df_imgs = pd.DataFrame(train_image_files)
 train_df_imgs.columns = ['image_filename']
train_imgs_pets = train_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
<feature_engineering>
train_data['Embarked']=train_data['Embarked'].fillna('S' )
avg_blur_result = Parallel(n_jobs=-1, verbose=1 )(
 delayed(get_average_color )(i)for i in train_df_imgs['image_filename'])
 train_df_imgs['average_red'] = [x[0] for x in avg_blur_result]
train_df_imgs['average_green'] = [x[1] for x in avg_blur_result]
train_df_imgs['average_blue'] = [x[2] for x in avg_blur_result]
train_df_imgs['blur'] = [x[3] for x in avg_blur_result]<feature_engineering>
train_data['Sex'] = pd.get_dummies(train_data['Sex'])
 test_data['Sex'] = pd.get_dummies(test_data['Sex'])
 train_data['Fare']=train_data['Fare'].astype('int32')
 train_data['Embarked'] = pd.factorize(train_data['Embarked'])[0]
test_data['Embarked'] = pd.factorize(test_data['Embarked'])[0]
test_data['Fare']=test_data['Fare'].fillna(test_data['Fare'].mean())
 test_data['Fare']=test_data['Fare'].astype('int32' )
w_h_average = Parallel(n_jobs=-1, verbose=1 )(
 delayed(average_pixel_width )(i)for i in train_df_imgs['image_filename'])
 train_df_imgs['width'] = [x[0] for x in w_h_average]
train_df_imgs['height'] = [x[1] for x in w_h_average]
train_df_imgs['resultion'] = train_df_imgs['width'] * train_df_imgs['height']
train_df_imgs['image_size'] = train_df_imgs['image_filename'].apply(getSize)
 train_df_imgs['ppi'] = train_df_imgs['resultion'] / train_df_imgs['image_size']<feature_engineering>
for i in range(0,len(train_data)) :
if train_data['Age'][i] <= 15:
train_data['Age'][i] = 0
elif(train_data['Age'][i] > 15)&(train_data['Age'][i] <=35):
train_data['Age'][i]=1
elif(train_data['Age'][i] > 35)&(train_data['Age'][i] <=55):
train_data['Age'][i]=2
elif(train_data['Age'][i] > 55)&(train_data['Age'][i] <=75):
train_data['Age'][i]=3
else:
train_data['Age'][i]=4
avg_blur_result = Parallel(n_jobs=-1, verbose=1 )(
 delayed(get_average_color )(i)for i in test_df_imgs['image_filename'])
 test_df_imgs['average_red'] = [x[0] for x in avg_blur_result]
test_df_imgs['average_green'] = [x[1] for x in avg_blur_result]
test_df_imgs['average_blue'] = [x[2] for x in avg_blur_result]
test_df_imgs['blur'] = [x[3] for x in avg_blur_result] 
<feature_engineering>
for i in range(0,len(test_data)) :
if test_data['Age'][i] <= 15:
test_data['Age'][i] = 0
elif(test_data['Age'][i] > 15)&(test_data['Age'][i] <=26):
test_data['Age'][i]=1
elif(test_data['Age'][i] > 35)&(test_data['Age'][i] <=55):
test_data['Age'][i]=2
elif(test_data['Age'][i] > 55)&(test_data['Age'][i] <=75):
test_data['Age'][i]=3
else:
test_data['Age'][i]=4
w_h_average = Parallel(n_jobs=-1, verbose=1 )(
 delayed(average_pixel_width )(i)for i in test_df_imgs['image_filename'])
 test_df_imgs['width'] = [x[0] for x in w_h_average]
test_df_imgs['height'] = [x[1] for x in w_h_average]
test_df_imgs['resultion'] = test_df_imgs['width'] * test_df_imgs['height']
test_df_imgs['image_size'] = test_df_imgs['image_filename'].apply(getSize)
 test_df_imgs['ppi'] = test_df_imgs['resultion'] / test_df_imgs['image_size']<define_variables>
for i in range(0,len(train_data)) :
if train_data['Fare'][i] <= 50:
train_data['Fare'][i] = 3
elif(train_data['Fare'][i] > 50)&(train_data['Fare'][i] <=150):
train_data['Fare'][i]=2
else:
train_data['Fare'][i]=1
aggs = {
'average_blue': ['sum', 'mean', 'var','min','max'],
'average_green': ['sum', 'mean', 'var','min','max'],
'average_red': ['sum', 'mean', 'var','min','max'],
'blur': ['sum', 'mean', 'var','min','max'],
'image_size': ['sum', 'mean', 'var','min','max'],
'width': ['sum', 'mean', 'var','min','max'],
'height': ['sum', 'mean', 'var','min','max'],
'ppi': ['sum', 'mean', 'var','min','max'],
'resultion': ['sum', 'mean', 'var','min','max'],
}
agg_train_imgs = train_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()
agg_test_imgs = test_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index()
agg_imgs = pd.concat([agg_train_imgs, agg_test_imgs], axis=0 ).reset_index(drop=True )<merge>
for i in range(0,len(test_data)) :
if test_data['Fare'][i] <= 50:
test_data['Fare'][i] = 3
elif(test_data['Fare'][i] > 50)&(test_data['Fare'][i] <=150):
test_data['Fare'][i]=2
else:
test_data['Fare'][i]=1
X_temp = X_temp.merge(agg_imgs, how='left', on='PetID' )<drop_column>
train_data['Fam'] = train_data['Parch'] + train_data['SibSp']
test_data['Fam'] = test_data['Parch'] + test_data['SibSp']
X_temp = X_temp.drop(to_drop_columns, axis=1 )<categorify>
for i in range(0,len(train_data)) :
if train_data['Fam'][i] == 0:
train_data['Fam'][i] = 0
elif(train_data['Fam'][i] >= 1)&(train_data['Fam'][i] <=3):
train_data['Fam'][i]=1
elif(train_data['Fam'][i] >= 4)&(train_data['Fam'][i] <=6):
train_data['Fam'][i]=2
elif(train_data['Fam'][i] >= 7)&(train_data['Fam'][i] <=9):
train_data['Fam'][i]=3
else:
train_data['Fam'][i]=4
for x in [
'Breed1','Breed2',
'main_breed_BreedName', 'second_breed_Type', 'second_breed_BreedName',
'breed_type_cross'
]:
X_temp = agg_helper(X_temp, x,'txt_pred')
 X_temp = agg_helper(X_temp, x,'img_pred')
 X_temp = agg_helper(X_temp, x,'tablur_pred')
 X_temp = agg_helper(X_temp, x,'height_mean')
 X_temp = agg_helper(X_temp, x,'Age')
 X_temp = agg_helper(X_temp, x,'image_size_mean')
 X_temp = agg_helper(X_temp, x,'resultion_mean')
 X_temp = agg_helper(X_temp, x,'ppi_mean')
<groupby>
for i in range(0,len(test_data)) :
if test_data['Fam'][i] == 0:
test_data['Fam'][i] = 0
elif(test_data['Fam'][i] >= 1)&(test_data['Fam'][i] <=3):
test_data['Fam'][i]=1
elif(test_data['Fam'][i] >= 4)&(test_data['Fam'][i] <=6):
test_data['Fam'][i]=2
elif(test_data['Fam'][i] >= 7)&(test_data['Fam'][i] <=9):
test_data['Fam'][i]=3
else:
test_data['Fam'][i]=4
for i in HD_cate:
if i in X_temp.columns:
X_temp[i] = X_temp.groupby(i)[i].transform('count' )<define_variables>
X = train_data[['Sex','Pclass','Age','Parch','Fam','Fare','Embarked']]
y = train_data[['Survived']]
X_test = test_data[['Sex','Pclass','Age','Parch','Fam','Fare','Embarked']]
train_X , test_X , train_y , test_y = train_test_split(X,y,test_size = 0.2,random_state=0 )
X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1 )<concatenate>
classifier = RandomForestClassifier(n_estimators=500,max_depth=3)
 classifier.fit(train_X,train_y)
 classifier.score(test_X,test_y )
len_train = len(X_train_non_null)
 data = X_train_non_null.append(X_test_non_null ).reset_index(drop=True )<import_modules>
xgb_model = xb.XGBClassifier(base_score=0.5,n_estimators=1000, learning_rate=0.05)
 xgb_model.fit(train_X, train_y, early_stopping_rounds=5,
eval_set=[(test_X, test_y)], verbose=False)
 xgb_model.score(test_X,test_y )
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<train_on_grid>
log_reg = LogisticRegression()
log_reg.fit(train_X,train_y)
 log_reg.score(test_X,test_y )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']<categorify>
predictions = xgb_model.predict(X_test )
<compute_train_metric><EOS> 
output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions})
 output.to_csv('submission.csv', index=False )