Kaggle competition: Titanic - Machine Learning from Disaster
def get_freq_encoding(dataset):
new_dataset = pd.DataFrame()
for c in dataset.columns:
data = dataset.groupby([c] ).size().reset_index()
new_dataset[c+"_freq"] = dataset[[c]].merge(data, on = c, how = "left")[0]
return new_dataset<categorify>
train_df=pd.read_csv('/kaggle/input/titanic/train.csv')
 test_df=pd.read_csv('/kaggle/input/titanic/test.csv')
 test_PassengerId=test_df['PassengerId']
new_train, new_test = transform_dataset(
 train[col4train], test[col4train],
get_freq_encoding 
)
 print(new_train.shape, new_test.shape)
 new_train.head(5 )<train_model>
train_df['Sex'].value_counts()
validate_model(
 model = get_model() ,
data = [new_train.values, y]
)<concatenate>
category2 = ['Ticket', 'Name', 'Cabin']
for c in category2:
print('{}
'.format(train_df[c].value_counts()))
new_train1, new_test1 = transform_dataset(
 train[col4train], test[col4train], get_freq_encoding 
)
 new_train2, new_test2 = transform_dataset(
 train[col4train], test[col4train], get_col_interactions_svd 
)
 new_train3, new_test3 = transform_dataset(
 train[col4train], test[col4train],
assign_rnd_integer, {"number_of_times":10} 
)
 new_train = pd.concat([new_train1, new_train2, new_train3], axis = 1)
 new_test = pd.concat([new_test1, new_test2, new_test3], axis = 1)
 print(new_train.shape, new_test.shape )<train_model>
train_df[['Pclass','Survived']].groupby('Pclass', as_index=False ).mean().sort_values(by='Survived', ascending=False )
validate_model(
 model = get_model() ,
data = [new_train.values, y]
)<save_to_csv>
train_df[['Sex','Survived']].groupby('Sex', as_index=False ).mean().sort_values(by='Survived', ascending=False )
model = get_model()
model.fit(new_train.values, y)
 predictions = model.predict_proba(new_test)[:,1]
submit = pd.DataFrame()
submit["Id"] = test["id"]
submit["ACTION"] = predictions
submit.to_csv("submission.csv", index = False )<set_options>
train_df[['SibSp','Survived']].groupby('SibSp', as_index=False ).mean().sort_values(by='Survived', ascending=False )
%matplotlib inline
%config InlineBackend.figure_format = 'svg'
warnings.filterwarnings("ignore")
 plt.rcParams['figure.figsize'] =(12, 9)
 plt.style.use('ggplot')
 pd.options.display.max_rows = 64
pd.options.display.max_columns = 512<load_from_csv>
train_df[['Parch','Survived']].groupby('Parch', as_index=False ).mean().sort_values(by='Survived', ascending=False )
train = pd.read_csv('.. /input/train/train.csv')
 train['AdoptionSpeed'].astype(np.int32)
 test = pd.read_csv('.. /input/test/test.csv')
 df = pd.concat([train,test],ignore_index=True )<define_variables>
def detect_outliers(df,columns):
outlier_list=[]
for c in columns:
Q1 = np.percentile(df[c],25)
 Q3 = np.percentile(df[c],75)
 IQR = Q3-Q1
outlier_step = 1.5*IQR
indices = df[(df[c] < Q1-outlier_step)|(df[c] > Q3+outlier_step)].index
outlier_list.extend(indices)
 outlier_list_counter=Counter(outlier_list)
 final_outlier_list = [i for i,v in outlier_list_counter.items() if v>2]
return final_outlier_list
train_sentiment_files = sorted(glob.glob('.. /input/train_sentiment/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/test_sentiment/*.json'))
sentimental_analysis = train_sentiment_files + test_sentiment_files<define_variables>
final_outlier_list = detect_outliers(train_df, ['Age', 'Fare', 'SibSp', 'Parch'])
 train_df.loc[final_outlier_list]
score_dict = dict(zip(petid,score))
magnitude_dict = dict(zip(petid,magnitude))<feature_engineering>
train_df = train_df.drop(final_outlier_list, axis=0 ).reset_index(drop=True )
df['Score'] = df['PetID'].map(score_dict)
 df['Score'][df.Score.isnull() ] = 0
df['Magnitude'] = df['PetID'].map(magnitude_dict)
 df['Magnitude'][df.Magnitude.isnull() ] = 0
df.set_index('PetID',inplace=True )<feature_engineering>
train_df_len = len(train_df)
 train_df = pd.concat([train_df,test_df], axis=0 ).reset_index(drop=True )
def namevaild(name):
if name == np.nan:
return 0
elif len(str(name)) < 3:
return 1
elif re.match(u'[0-9]', str(name ).lower()):
return 1
elif len(set(str(name ).lower().split(' ')+['no','not','yet','male','female','unnamed'])) != len(set(str(name ).lower().split(' ')))+6:
return 1
else:
return 2
df['Name_state'] = df['Name'].apply(namevaild )<categorify>
train_df.columns[train_df.isnull().any() ]
df['Fee_per_pet'] = df.Fee/df.Quantity
df['Fee_Bin']=pd.factorize(pd.cut(df.Fee_per_pet,bins=[0,0.01,50,100,200,500,3000],right=False)) [0]
fee_bin_dummies_df = pd.get_dummies(df['Fee_Bin'] ).rename(columns=lambda x: 'Fee_Bin_' + str(x))
df = pd.concat([df, fee_bin_dummies_df], axis=1 )<categorify>
train_df.isnull().sum()
df['Quantity_Bin']=pd.factorize(pd.cut(df.Quantity,bins=[1,2,4,22],right=False)) [0]
quantity_bin_dummies_df = pd.get_dummies(df['Quantity_Bin'] ).rename(columns=lambda x: 'Quantity_Bin_' + str(x))
df = pd.concat([df, quantity_bin_dummies_df], axis=1 )<categorify>
train_df[train_df['Embarked'].isnull() == True]
df.VideoAmt = df.VideoAmt.apply(lambda x: 1 if x > 0 else 0)
 df['PhotoAmt_Bin']=pd.factorize(pd.cut(df.PhotoAmt,bins=[0,1,2,4,31],right=False)) [0]
photo_bin_dummies_df = pd.get_dummies(df['PhotoAmt_Bin'] ).rename(columns=lambda x: 'PhotoAmt_Bin_' + str(x))
df = pd.concat([df, photo_bin_dummies_df], axis=1 )<categorify>
train_df['Embarked'][train_df['Pclass']==1].value_counts()
def map_state(state):
if state == 41326:
return 'Selangor'
elif state == 41401:
return 'Kuala_Lumpur'
else:
return 'Other_State'
df['State_Bin'] = df.State.apply(map_state)
 state_bin_dummies_df = pd.get_dummies(df['State_Bin'] ).rename(columns=lambda x: 'State_' + str(x))
df = pd.concat([df, state_bin_dummies_df], axis=1 )<categorify>
train_df['Embarked'] = train_df['Embarked'].fillna('C' )
rescuer_dict = df.RescuerID.value_counts().to_dict()
df['Rescuer_Num'] = df.RescuerID.map(rescuer_dict)
<load_from_csv>
train_df[train_df['Embarked'].isnull() == True]
breeds = pd.read_csv('.. /input/breed_labels.csv')
 breeds_dict = {k: v for k, v in zip(breeds['BreedID'], breeds['BreedName'])}
df['Breed1_name'] = df['Breed1'].apply(lambda x: '_'.join(breeds_dict[x].split())if x in breeds_dict else 'NA')
 df['Breed2_name'] = df['Breed2'].apply(lambda x: '_'.join(breeds_dict[x].split())if x in breeds_dict else 'NA' )<feature_engineering>
train_df[train_df['Fare'].isnull() ==True]
df['Breed'] = df['Breed1_name'] + '--' + df['Breed2_name']
def mix_breed(string):
breed = string.split('--')
 if breed[0] in ['Mixed_Breed','NA']:
return 1
elif breed[1] == 'Mixed_Breed':
return 1
elif breed[1] == 'NA':
return 0
elif breed[0] != breed[1]:
return 1
else:
return 0
df['Mixed_Breed'] = df.Breed.apply(mix_breed )<categorify>
train_df['Fare'] = train_df['Fare'].fillna(train_df['Fare'][train_df.Pclass==3].mean() )
<filter>
train_df[train_df['Fare'].isnull() ==True]
df_copy = df.drop(columns=['Description','Fee','Fee_per_pet','Name','PhotoAmt','Quantity','RescuerID','State','State_Bin','Fee_Bin','Quantity_Bin','PhotoAmt_Bin','Breed','Breed1_name','Breed2_name'])
 train = df_copy[df.AdoptionSpeed.notnull() ]
test = df_copy[df.AdoptionSpeed.isnull() ]
print(train.shape, test.shape )<prepare_x_and_y>
train_df[train_df['Age'].isnull() ]
X_train = train.drop(columns=['AdoptionSpeed'])
 y_train = train.AdoptionSpeed
X_test = test.drop(columns=['AdoptionSpeed'] )<train_model>
train_df['Sex']=list(1 if i=='male' else 0 for i in train_df['Sex'] )
rf = RandomForestClassifier(n_estimators = 600, max_depth=None, criterion='gini')
 rf.fit(X_train,y_train)
 y_predict = rf.predict(X_test ).astype(np.int32)
 submission = pd.DataFrame({'PetID': test.index, 'AdoptionSpeed': y_predict})
 submission = submission[['PetID','AdoptionSpeed']]
submission.head()<save_to_csv>
train_df['Sex']=list('male' if i==1 else 'female' for i in train_df['Sex'] )
submission.to_csv('submission.csv', index=False )<save_to_csv>
index_nan_age = list(train_df[train_df['Age'].isnull() ].index )
submission.to_csv('submission.csv', index=False )<set_options>
for i in index_nan_age:
age_prediction = train_df['Age'][(( train_df['SibSp']==train_df.iloc[i]['SibSp'])&(train_df['Parch']==train_df.iloc[i]['Parch'])&(train_df['Pclass']==train_df.iloc[i]['Pclass'])) ].median()
age_med = train_df['Age'].median()
if not np.isnan(age_prediction):
train_df['Age'].iloc[i] = age_prediction
else:
train_df['Age'].iloc[i] = age_med
%matplotlib inline
np.random.seed(seed=1337)
 warnings.filterwarnings('ignore')
 split_char = '/'<load_from_csv>
train_df[train_df['Age'].isnull() ]
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv' )<import_modules>
from sklearn.tree import DecisionTreeRegressor
import cv2
import os
from keras.applications.densenet import preprocess_input, DenseNet121<define_variables>
fare = np.array(train_df.Fare[:train_df_len] ).reshape(-1,1)
 survived = np.array(train_df.Survived[:train_df_len] ).reshape(-1,1)
 DecTree = DecisionTreeRegressor(max_leaf_nodes=150)
 DecTree.fit(fare, survived)
 train_df['Fare_class'] = DecTree.apply(np.array(train_df.Fare ).reshape(-1,1))
img_size = 256
batch_size = 32<define_search_model>
train_df = pd.get_dummies(data=train_df, columns=['Fare_class'] )
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights=".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )<define_variables>
train_df.drop('Fare', axis=1, inplace=True )
pet_ids = train['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]<create_dataframe>
train_df['Age'] = train_df['Age'].astype('int' )
train_feats = pd.DataFrame.from_dict(features, orient='index')
 train_feats.columns = [f'pic_{i}' for i in range(train_feats.shape[1])]<define_variables>
train_df['Age_based_survival'] = 0
pet_ids = test['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/test_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]<create_dataframe>
for i in range(train_df['Age'].max() +1):
df = train_df[train_df['Age']==i][train_df.Survived.isna() == False]
avg_surv_prob = df['Survived'].mean()
train_df.loc[train_df.Age==i,'Age_based_survival'] = avg_surv_prob
test_feats = pd.DataFrame.from_dict(features, orient='index')
 test_feats.columns = [f'pic_{i}' for i in range(test_feats.shape[1])]<rename_columns>
train_df['Age_based_survival'] = train_df['Age_based_survival'].fillna(0)
 train_df[train_df['Age_based_survival'].isnull() ]
train_feats = train_feats.reset_index()
train_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 test_feats = test_feats.reset_index()
test_feats.rename({'index': 'PetID'}, axis='columns', inplace=True )<concatenate>
train_df.drop('Age', axis=1, inplace=True)
 train_df.head()
all_ids = pd.concat([train, test], axis=0, ignore_index=True, sort=False)[['PetID']]
all_ids.shape<concatenate>
train_df['Title'] = [i.split('.')[0].split(',')[-1].strip() for i in train_df['Name']]
train_df['Title']
n_components = 32
svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 features_df = pd.concat([train_feats, test_feats], axis=0)
 features = features_df[[f'pic_{i}' for i in range(256)]].values
svd_col = svd_.fit_transform(features)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('IMG_SVD_')
 img_features = pd.concat([all_ids, svd_col], axis=1 )<load_from_csv>
train_df['Title'] = train_df['Title'].replace(['Lady','the Countess','Capt','Col','Don','Dr','Major','Rev','Sir','Jonkheer','Dona'],'other' )
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )<define_variables>
train_df['Title'] = [0 if i=='Master' else 1 if i=='Miss' or i=='Ms' or i=='Mlle' or i=='Mrs' else 2 if i=='Mr' else 3 for i in train_df['Title']]
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print(f'num of train images files: {len(train_image_files)}')
 print(f'num of train metadata files: {len(train_metadata_files)}')
 print(f'num of train sentiment files: {len(train_sentiment_files)}')
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print(f'num of test images files: {len(test_image_files)}')
 print(f'num of test metadata files: {len(test_metadata_files)}')
 print(f'num of test sentiment files: {len(test_sentiment_files)}' )<load_pretrained>
train_df.drop('Name', axis=1, inplace=True )
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_json_file(self, filename):
with open(filename, 'r', encoding='utf-8')as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns')
 file_sentences_sentiment_df = pd.DataFrame(
 {
'magnitude_sum': file_sentences_sentiment['magnitude'].sum(axis=0),
'score_sum': file_sentences_sentiment['score'].sum(axis=0),
'magnitude_mean': file_sentences_sentiment['magnitude'].mean(axis=0),
'score_mean': file_sentences_sentiment['score'].mean(axis=0),
'magnitude_var': file_sentences_sentiment['magnitude'].var(axis=0),
'score_var': file_sentences_sentiment['score'].var(axis=0),
}, index=[0] 
)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
df_sentiment = pd.concat([df_sentiment, file_sentences_sentiment_df], axis=1)
 df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = f'.. /input/petfinder-adoption-prediction/{mode}_sentiment/{pet_id}.json'
try:
sentiment_file = pet_parser.open_json_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob(f'.. /input/petfinder-adoption-prediction/{mode}_metadata/{pet_id}*.json'))
if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_json_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()<categorify>
train_df = pd.get_dummies(data=train_df, columns=['Title'] )
aggregates = ['sum', 'mean', 'var']
sent_agg = ['sum']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 train_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 test_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()<merge>
train_df['Fsize'] = train_df['SibSp'] + train_df['Parch'] + 1
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]<merge>
train_df['Family_size'] = [1 if i in [2,3,4] else 0 for i in train_df['Fsize']]
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )<concatenate>
train_df['SibSp_categ'] = [0 if i in [0,1,2] else 1 if i in [3,4] else 2 for i in train_df.SibSp]
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False )<define_variables>
train_df = pd.get_dummies(data=train_df, columns=['SibSp_categ'] )
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID']<merge>
train_df['Parch_categ'] = [0 if i in [1,2,3] else 1 if i in [0,5] else 2 for i in train_df.Parch]
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID' )<feature_engineering>
train_df = pd.get_dummies(data=train_df, columns=['Parch_categ'] )
for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]<feature_engineering>
train_df = pd.get_dummies(data=train_df, columns=['Embarked'] )
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('none' )<categorify>
ticket_list=[]
for i in train_df['Ticket']:
if not i.strip().isdigit() :
ticket_list.append(i.strip().replace('.','' ).replace('/','' ).split() [0])
 else:
ticket_list.append('x')
 train_df['Ticket'] = ticket_list
X_temp['Length_Description'] = X_text['Description'].map(len)
 X_temp['Length_metadata_annots_top_desc'] = X_text['metadata_annots_top_desc'].map(len)
 X_temp['Lengths_sentiment_entities'] = X_text['sentiment_entities'].map(len )<feature_engineering>
train_df = pd.get_dummies(train_df, columns=['Ticket'], prefix='T' )
n_components = 16
text_features = []
for i in X_text.columns:
print(f'generating features from: {i}')
 tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_{}_'.format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )<merge>
train_df['Pclass'] = train_df['Pclass'].astype('category')
 train_df = pd.get_dummies(data=train_df, columns=['Pclass'])
 train_df.head()
X_temp = X_temp.merge(img_features, how='left', on='PetID' )<define_variables>
train_df['Sex'] = train_df['Sex'].astype('category')
 train_df = pd.get_dummies(data=train_df, columns=['Sex'])
 train_df.head()
train_df_ids = train[['PetID']]
test_df_ids = test[['PetID']]
train_df_imgs = pd.DataFrame(train_image_files)
 train_df_imgs.columns = ['image_filename']
train_imgs_pets = train_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 def getSize(filename):
st = os.stat(filename)
 return st.st_size
def getDimensions(filename):
img_size = Image.open(filename ).size
return img_size
train_df_imgs['image_size'] = train_df_imgs['image_filename'].apply(getSize)
 train_df_imgs['temp_size'] = train_df_imgs['image_filename'].apply(getDimensions)
 train_df_imgs['width'] = train_df_imgs['temp_size'].apply(lambda x : x[0])
 train_df_imgs['height'] = train_df_imgs['temp_size'].apply(lambda x : x[1])
 train_df_imgs = train_df_imgs.drop(['temp_size'], axis=1)
 test_df_imgs['image_size'] = test_df_imgs['image_filename'].apply(getSize)
 test_df_imgs['temp_size'] = test_df_imgs['image_filename'].apply(getDimensions)
 test_df_imgs['width'] = test_df_imgs['temp_size'].apply(lambda x : x[0])
 test_df_imgs['height'] = test_df_imgs['temp_size'].apply(lambda x : x[1])
 test_df_imgs = test_df_imgs.drop(['temp_size'], axis=1)
 aggs = {
'image_size': ['sum', 'mean', 'var'],
'width': ['sum', 'mean', 'var'],
'height': ['sum', 'mean', 'var'],
}
agg_train_imgs = train_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()
agg_test_imgs = test_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index()
agg_imgs = pd.concat([agg_train_imgs, agg_test_imgs], axis=0 ).reset_index(drop=True )<merge>
train_df.drop(labels=['PassengerId', 'Cabin'], axis=1, inplace=True)
 train_df.columns
X_temp = X_temp.merge(agg_imgs, how='left', on='PetID' )<drop_column>
from sklearn.model_selection import train_test_split, StratifiedKFold, GridSearchCV
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier, VotingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score
X_temp = X_temp.drop(to_drop_columns, axis=1 )<define_variables>
test = train_df[train_df_len:]
test.drop(labels=['Survived'], axis=1, inplace=True)
 test.head()
X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1 )<count_missing_values>
train = train_df[:train_df_len]
y_train = train['Survived']
x_train = train.drop('Survived', axis=1)
 x_train,x_test,y_train,y_test = train_test_split(x_train,y_train,test_size=0.2, random_state=42)
 print('x_train: ',len(x_train))
print('x_test: ',len(x_test))
print('y_train: ',len(y_train))
print('y_test: ',len(y_test))
print('test: ',len(test))
X_train_non_null.isnull().any().any() , X_test_non_null.isnull().any().any()<import_modules>
log_reg = LogisticRegression()
log_reg.fit(x_train,y_train)
 acc_logreg_train = round(log_reg.score(x_train,y_train)*100,2)
 acc_logreg_test = round(log_reg.score(x_test,y_test)*100,2)
 print('Training accuracy: %{}'.format(acc_logreg_train))
print('Testing accuracy: %{}'.format(acc_logreg_test))
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<train_on_grid>
random_state = 42
model_list = [DecisionTreeClassifier(random_state = random_state),
SVC(random_state = random_state, probability=True),
RandomForestClassifier(random_state = random_state),
LogisticRegression(random_state = random_state),
KNeighborsClassifier() ]
dt_param_grid = {"min_samples_split" : range(10,500,20),
"max_depth": range(1,20,2)}
svc_param_grid = {"kernel" : ["rbf"],
"gamma": [0.001, 0.01, 0.1, 1],
"C": [1,10,50,100,200,300,1000]}
rf_param_grid = {"max_features": [1,3,10],
"min_samples_split":[2,3,10],
"min_samples_leaf":[1,3,10],
"bootstrap":[False],
"n_estimators":[100,300],
"criterion":["gini"]}
logreg_param_grid = {"C":np.logspace(-3,3,7),
"penalty": ["l1","l2"]}
knn_param_grid = {"n_neighbors": np.linspace(1,19,10, dtype = int ).tolist() ,
"weights": ["uniform","distance"],
"metric":["euclidean","manhattan"]}
grid_list = [dt_param_grid, svc_param_grid, rf_param_grid, logreg_param_grid, knn_param_grid]
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']<init_hyperparams>
score_list = []
best_estimators = []
for i in range(5):
my_model = GridSearchCV(estimator=model_list[i], param_grid=grid_list[i], scoring='accuracy', n_jobs=-1, cv=StratifiedKFold(n_splits=10))
my_model.fit(x_train, y_train)
 score_list.append(accuracy_score(my_model.best_estimator_.predict(x_test), y_test))
best_estimators.append(my_model.best_estimator_)
 print('Best scores: ', score_list, '
' )
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'eta': 0.0123,
'subsample': 0.8,
'colsample_bytree': 0.85,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
}<prepare_x_and_y>
votingC = VotingClassifier([('RandomForest',best_estimators[2]), 
('SVC',best_estimators[1]), 
('Logistic',best_estimators[3])], voting='soft', n_jobs=-1)
 votingC.fit(x_train, y_train)
 print('Ensembled score: ', accuracy_score(votingC.predict(x_test),y_test))
def run_xgb(params, X_train, X_test):
n_splits = 5
verbose_eval = 1000
num_rounds = 30000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test<train_model>
test_survived = pd.Series(votingC.predict(test), name='Survived' ).astype(int)
 results = pd.concat([test_PassengerId, test_survived], axis=1)
 results.to_csv('titanic.csv', index=False )