Kaggle competition: Titanic - Machine Learning from Disaster
nfolds=5
folds = StratifiedKFold(n_splits=nfolds,shuffle=True, random_state=15)
 avg_train_kappa = 0
avg_valid_kappa = 0
batch_size=1000
coeffs=None
x_test = get_keras_data(test_df, desc_embs[len(train_df):])
 adoptions_keras = np.zeros(( len(test_df),))
oof_train_keras = np.zeros(( train_df.shape[0]))
i =0
for train_idx, valid_idx in folds.split(train_df[cat_vars+cont_vars], train_df["AdoptionSpeed"]):
x_train = get_keras_data(train_df.iloc[train_idx], desc_embs[train_idx])
 x_valid = get_keras_data(train_df.iloc[valid_idx], desc_embs[valid_idx])
 y_train, y_valid = train_df["AdoptionSpeed"][train_idx].values, train_df["AdoptionSpeed"][valid_idx].values
model = get_model()
clr_tri = CyclicLR(base_lr=2e-3, max_lr=4e-2, step_size=len(train_df)//batch_size, mode="triangular2")
 qwk_eval = QWKEvaluation(validation_data=(x_valid, y_valid), interval=1)
 history = model.fit(x_train, y_train, batch_size=batch_size, validation_data=(x_valid, y_valid),
epochs=20, callbacks=[clr_tri, qwk_eval])
 model.load_weights('checkpoint.h5')
 y_train_pred = eval_predict(y_train, model.predict(x_train, batch_size=1000))
y_valid_pred = eval_predict(y_valid, model.predict(x_valid, batch_size=1000))
avg_train_kappa += quadratic_weighted_kappa(y_train_pred, y_train)
 avg_valid_kappa += quadratic_weighted_kappa(y_valid_pred, y_valid)
 coeffs = eval_predict(y_valid, model.predict(x_valid, batch_size=1000), ret_coeffs=True)
 adoptions_keras += model.predict(x_test, batch_size=batch_size ).reshape(-1,)
 oof_train_keras[valid_idx] = y_valid_pred
print("
avg train kappa:", avg_train_kappa/nfolds,)
 print("
avg valid kappa:", avg_valid_kappa/nfolds, )<count_values>
training = pd.read_csv('/kaggle/input/titanic/train.csv')
 test = pd.read_csv('/kaggle/input/titanic/test.csv')
 training['train_test'] = 1
test['train_test'] = 0
test['Survived'] = np.NaN
all_data = pd.concat([training,test])
Counter(X_train['AdoptionSpeed'] )<count_values>
df_num = training[['Age','SibSp', 'Parch', 'Fare']]
df_cat = training[['Survived', 'Pclass','Sex','Ticket', 'Cabin', 'Embarked']]
Counter(X_train['AdoptionSpeed'] )<predict_on_test>
pd.pivot_table(training, index = 'Survived', values = ['Age','Fare','SibSp','Parch'] )
coeffs[0] = 1.69
coeffs[1] = 2.2
coeffs[3] = 2.9
print('True Counter',Counter(X_train['AdoptionSpeed']))
train_pred_keras = eval_predict(y_pred=oof_train_keras, coeffs=list(coeffs)).astype(int)
 print('Train pred Counter',Counter(train_pred_keras))
test_pred_keras = eval_predict(y_pred=adoptions_keras/nfolds, coeffs=list(coeffs)).astype(int)
 print('Test pred Counter',Counter(test_pred_keras))<concatenate>
for i in ['Pclass', 'Sex', 'Embarked']:
print(pd.pivot_table(training, index = 'Survived', columns = i, values = 'Ticket', aggfunc = 'count'), '
' )
train_stack = np.vstack([train_pred_keras, train_predictions_lgb] ).transpose()
test_stack = np.vstack([test_pred_keras, test_predictions_lgb] ).transpose()<import_modules>
df_cat.Cabin
training['cabin_multiple'] = training.Cabin.apply(lambda x: 0 if pd.isna(x)else len(x.split(' ')))
 training['cabin_multiple'].value_counts()
from sklearn.linear_model import Ridge<prepare_x_and_y>
pd.pivot_table(training, index = 'Survived', columns = 'cabin_multiple', values = 'Ticket', aggfunc = 'count' )
folds = StratifiedKFold(n_splits=10, shuffle=True, random_state=15)
 oof = np.zeros(train_stack.shape[0])
 predictions = np.zeros(test_stack.shape[0])
 qwk_scores = []
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_stack, X_train['AdoptionSpeed'].values)) :
trn_data, trn_y = train_stack[trn_idx], X_train['AdoptionSpeed'].iloc[trn_idx].values
val_data, val_y = train_stack[val_idx], X_train['AdoptionSpeed'].iloc[val_idx].values
clf = Ridge(alpha=1)
 clf.fit(trn_data, trn_y)
 val_pred = clf.predict(val_data)
 pred_val_y_k = eval_predict(y_pred=val_pred, coeffs=list(coeffs)).astype(int)
 print("Valid Counts = ", Counter(val_y))
print("Predicted Counts = ", Counter(pred_val_y_k))
qwk = quadratic_weighted_kappa(val_y, pred_val_y_k)
 qwk_scores.append(qwk)
 print("QWK = ", qwk)
 oof[val_idx] = clf.predict(val_data)
 predictions += clf.predict(test_stack)/ folds.n_splits
print('{} cv mean QWK score : {}'.format('Stacked_model_ridge', np.mean(qwk_scores)) )<predict_on_test>
training['cabin_adv'] = training.Cabin.apply(lambda x : str(x)[0] )
pred_final = eval_predict(y_pred=predictions, coeffs=list(coeffs)).astype(int )<count_values>
print(training.cabin_adv.value_counts())
 pd.pivot_table(training, index='Survived', columns='cabin_adv', values='Name', aggfunc='count' )
Counter(pred_final )<count_values>
training['numeric_tickets'] = training.Ticket.apply(lambda x : 1 if x.isnumeric() else 0)
 training['ticket_letters'] = training.Ticket.apply(lambda x : ''.join(x.split(' ')[:-1] ).replace('.','' ).replace('/','' ).lower()
if len(x.split(' ')[:-1])> 0 else 0)
 print(training['numeric_tickets'].value_counts() )
Counter(test_pred_keras )<count_values>
pd.pivot_table(training, index='Survived', columns='numeric_tickets', values='Name', aggfunc='count' )
Counter(test_predictions_lgb )<save_to_csv>
training['ticket_letters'].value_counts()
submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': pred_final.astype(np.int32)})
 submission.head()
submission.to_csv('submission.csv', index=False )<import_modules>
training.Name.head(50)
 training['name_title'] = training.Name.apply(lambda x : x.split(',')[1].split('.')[0].strip())
 training['name_title'].value_counts()
def kappa(y_true, y_pred):
return cohen_kappa_score(y_true, y_pred, weights='quadratic')
 def warn(*args, **kwargs):
pass
warnings.warn = warn
%matplotlib inline
pd.options.display.max_rows = 128
pd.options.display.max_columns = 128<set_options>
all_data['cabin_multiple'] = all_data.Cabin.apply(lambda x: 0 if pd.isna(x)else len(x.split(' ')))
 all_data['cabin_adv'] = all_data.Cabin.apply(lambda x : str(x)[0])
 all_data['numeric_tickets'] = all_data.Ticket.apply(lambda x : 1 if x.isnumeric() else 0)
 all_data['ticket_letters'] = all_data.Ticket.apply(lambda x : ''.join(x.split(' ')[:-1] ).replace('.','')
.replace('/','' ).lower() if len(x.split(' ')[:-1])> 0 else 0)
 all_data['name_title'] = all_data.Name.apply(lambda x : x.split(',')[1].split('.')[0].strip())
 all_data.Age = all_data.Age.fillna(training.Age.mean())
 all_data.Fare = all_data.Fare.fillna(training.Fare.mean())
 all_data.dropna(subset=['Embarked'], inplace=True)
 all_data['norm_fare'] = np.log(all_data.Fare+1)
 plt.hist(all_data.Fare, bins=50)
 plt.title('Fare - Antes da normalização')
 plt.show()
plt.hist(all_data.norm_fare, bins=50)
 plt.title('Fare - Depois da normalização')
 plt.show()
all_data.Pclass = all_data.Pclass.astype(str)
 all_dummies = pd.get_dummies(all_data[['Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'norm_fare', 'Embarked',
'cabin_adv', 'cabin_multiple','numeric_tickets','name_title', 'train_test']])
 x_train = all_dummies[all_dummies.train_test == 1].drop(['train_test'], axis = 1)
 x_test = all_dummies[all_dummies.train_test == 1].drop(['train_test'], axis = 1)
 y_train = all_data[all_data.train_test == 1].Survived
y_train.shape
plt.rcParams['figure.figsize'] =(12, 9 )<load_from_csv>
Scale = StandardScaler()
all_dummies_scaled = all_dummies.copy()
all_dummies_scaled[['Age', 'SibSp', 'Parch', 'norm_fare']] = Scale.fit_transform(all_dummies_scaled[['Age', 'SibSp', 'Parch', 'norm_fare']])
 x_train_scaled = all_dummies_scaled[all_dummies_scaled.train_test == 1].drop(['train_test'], axis = 1)
 x_test_scaled = all_dummies_scaled[all_dummies_scaled.train_test == 1].drop(['train_test'], axis = 1)
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv' )<load_from_csv>
from sklearn.model_selection import cross_validate
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import LogisticRegression
from sklearn import tree
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )<load_from_csv>
classifiers = {}
train_img = pd.read_csv(".. /input/extract-image-features-from-pretrained-nn/train_img_features.csv")
 test_img = pd.read_csv(".. /input/extract-image-features-from-pretrained-nn/test_img_features.csv")
 train_img.rename(columns=lambda i: f"img_{i}" ,inplace=True)
 test_img.rename(columns=lambda i: f"img_{i}" ,inplace=True)
 train = pd.concat([train, train_img], axis=1)
 test = pd.concat([test, test_img], axis=1 )<load_from_disk>
gnb = GaussianNB()
cv = cross_validate(gnb, x_train, y_train, cv = 5, return_train_score = True, return_estimator = True)
 classifiers['GaussianNBNotScaled'] = cv
print('Average performance on training set:
')
 print(cv['train_score'].mean())
 print('
Average performance on test set:
')
 print(cv['test_score'].mean() )
with open('.. /input/cat-and-dog-breeds-parameters/rating.json', 'r')as f:
ratings = json.load(f )<feature_engineering>
gnb = GaussianNB()
cv = cross_validate(gnb, x_train_scaled, y_train, cv = 5, return_train_score = True, return_estimator = True)
 classifiers['GaussianNBScaled'] = cv
print('Average performance on training set:
')
 print(cv['train_score'].mean())
 print('
Average performance on test set:
')
 print(cv['test_score'].mean() )
breed_id = {}
for id,name in zip(labels_breed.BreedID,labels_breed.BreedName):
breed_id[id] = name<create_dataframe>
lr = LogisticRegression(max_iter = 2000)
 cv = cross_validate(lr, x_train, y_train, cv = 5, return_train_score = True, return_estimator = True)
 classifiers['LogisticRegressionNotScaled'] = cv
print('Average performance on training set:
')
 print(cv['train_score'].mean())
 print('
Average performance on test set:
')
 print(cv['test_score'].mean() )
breed_ratings = json.load(open(".. /input/cat-and-dog-breeds-parameters/rating.json",'r'))
species_keys = list(breed_ratings.keys())
 breed_ratings_dict = {**breed_ratings['cat_breeds']}
breed_ratings_dict = {**breed_ratings_dict,**breed_ratings['dog_breeds']}
breed_score_df = pd.DataFrame(breed_ratings_dict ).T.reset_index().rename(columns={'index':'breed_name'})
<define_variables>
lr = LogisticRegression(max_iter = 2000)
 cv = cross_validate(lr, x_train_scaled, y_train, cv = 5, return_train_score = True, return_estimator = True)
 classifiers['LogisticRegressionScaled'] = cv
print('Average performance on training set:
')
 print(cv['train_score'].mean())
 print('
Average performance on test set:
')
 print(cv['test_score'].mean() )
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print('num of train images files: {}'.format(len(train_image_files)))
 print('num of train metadata files: {}'.format(len(train_metadata_files)))
 print('num of train sentiment files: {}'.format(len(train_sentiment_files)))
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print('num of test images files: {}'.format(len(test_image_files)))
 print('num of test metadata files: {}'.format(len(test_metadata_files)))
 print('num of test sentiment files: {}'.format(len(test_sentiment_files)) )<count_unique_values>
dt = tree.DecisionTreeClassifier(random_state = 10)
 cv = cross_validate(dt,x_train,y_train,cv=5, return_train_score = True, return_estimator = True)
 classifiers['DecisionTreeNotScaled'] = cv
print('Average performance on training set:
')
 print(cv['train_score'].mean())
 print('
Average performance on test set:
')
 print(cv['test_score'].mean() )
test_df_ids = test[['PetID']]
print(test_df_ids.shape)
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 print(len(test_imgs_pets.unique()))
pets_with_images = len(np.intersect1d(test_imgs_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with images: {:.3f}'.format(pets_with_images / test_df_ids.shape[0]))
test_df_ids = test[['PetID']]
test_df_metadata = pd.DataFrame(test_metadata_files)
 test_df_metadata.columns = ['metadata_filename']
test_metadata_pets = test_df_metadata['metadata_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 test_df_metadata = test_df_metadata.assign(PetID=test_metadata_pets)
 print(len(test_metadata_pets.unique()))
pets_with_metadatas = len(np.intersect1d(test_metadata_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with metadata: {:.3f}'.format(pets_with_metadatas / test_df_ids.shape[0]))
test_df_ids = test[['PetID']]
test_df_sentiment = pd.DataFrame(test_sentiment_files)
 test_df_sentiment.columns = ['sentiment_filename']
test_sentiment_pets = test_df_sentiment['sentiment_filename'].apply(lambda x: x.split('/')[-1].split('.')[0])
 test_df_sentiment = test_df_sentiment.assign(PetID=test_sentiment_pets)
 print(len(test_sentiment_pets.unique()))
pets_with_sentiments = len(np.intersect1d(test_sentiment_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with sentiment: {:.3f}'.format(pets_with_sentiments / test_df_ids.shape[0]))
print('images and metadata distributions the same? {}'.format(
 np.all(test_metadata_pets == test_imgs_pets)) )<load_pretrained>
dt = tree.DecisionTreeClassifier(random_state = 10)
 cv = cross_validate(dt,x_train_scaled,y_train,cv=5, return_train_score = True, return_estimator = True)
 classifiers['DecisionTreeScaled'] = cv
print('Average performance on training set:
')
 print(cv['train_score'].mean())
 print('
Average performance on test set:
')
 print(cv['test_score'].mean() )
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_metadata_file(self, filename):

 with open(filename, 'r')as f:
metadata_file = json.load(f)
 return metadata_file
def open_sentiment_file(self, filename):

 with open(filename, 'r')as f:
sentiment_file = json.load(f)
 return sentiment_file
def open_image_file(self, filename):

 image = np.asarray(Image.open(filename))
return image
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 if self.extract_sentiment_text:
file_sentences_text = [x['text']['content'] for x in file['sentences']]
file_sentences_text = self.sentence_sep.join(file_sentences_text)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns' ).sum()
file_sentences_sentiment = file_sentences_sentiment.add_prefix('document_' ).to_dict()
file_sentiment.update(file_sentences_sentiment)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
if self.extract_sentiment_text:
df_sentiment['text'] = file_sentences_text
df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations'][:int(len(file['labelAnnotations'])* 0.3)]
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = '.. /input/petfinder-adoption-prediction/{}_sentiment/{}.json'.format(mode, pet_id)
 try:
sentiment_file = pet_parser.open_sentiment_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob('.. /input/petfinder-adoption-prediction/{}_metadata/{}*.json'.format(mode, pet_id)))
 if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_metadata_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()<categorify>
knn = KNeighborsClassifier()
cv = cross_validate(knn,x_train,y_train,cv=5, return_train_score = True, return_estimator = True)
 classifiers['KNeighborsNotScaled'] = cv
print('Average performance on training set:
')
 print(cv['train_score'].mean())
 print('
Average performance on test set:
')
 print(cv['test_score'].mean() )
def impact_coding(data, feature, target='y'):

 n_folds = 20
n_inner_folds = 10
impact_coded = pd.Series()
oof_default_mean = data[target].mean()
kf = KFold(n_splits=n_folds, shuffle=True)
 oof_mean_cv = pd.DataFrame()
split = 0
for infold, oof in kf.split(data[feature]):
impact_coded_cv = pd.Series()
kf_inner = KFold(n_splits=n_inner_folds, shuffle=True)
 inner_split = 0
inner_oof_mean_cv = pd.DataFrame()
oof_default_inner_mean = data.iloc[infold][target].mean()
for infold_inner, oof_inner in kf_inner.split(data.iloc[infold]):
oof_mean = data.iloc[infold_inner].groupby(by=feature)[target].mean()
impact_coded_cv = impact_coded_cv.append(data.iloc[infold].apply(
 lambda x: oof_mean[x[feature]]
if x[feature] in oof_mean.index
else oof_default_inner_mean
, axis=1))
inner_oof_mean_cv = inner_oof_mean_cv.join(pd.DataFrame(oof_mean), rsuffix=inner_split, how='outer')
 inner_oof_mean_cv.fillna(value=oof_default_inner_mean, inplace=True)
 inner_split += 1
oof_mean_cv = oof_mean_cv.join(pd.DataFrame(inner_oof_mean_cv), rsuffix=split, how='outer')
 oof_mean_cv.fillna(value=oof_default_mean, inplace=True)
 split += 1
impact_coded = impact_coded.append(data.iloc[oof].apply(
 lambda x: inner_oof_mean_cv.loc[x[feature]].mean()
if x[feature] in inner_oof_mean_cv.index
else oof_default_mean
, axis=1))
return impact_coded, oof_mean_cv.mean(axis=1), oof_default_mean
def frequency_encoding(df, col_name):
new_name = "{}_counts".format(col_name)
 new_col_name = "{}_freq".format(col_name)
 grouped = df.groupby(col_name ).size().reset_index(name=new_name)
 df = df.merge(grouped, how = "left", on = col_name)
 df[new_col_name] = df[new_name]/df[new_name].count()
del df[new_name]
return df 
<categorify>
knn = KNeighborsClassifier()
cv = cross_validate(knn,x_train_scaled,y_train,cv=5, return_train_score = True, return_estimator = True)
 classifiers['KNeighborsScaled'] = cv
print('Average performance on training set:
')
 print(cv['train_score'].mean())
 print('
Average performance on test set:
')
 print(cv['test_score'].mean() )
aggregates = ['mean', 'sum', 'var']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(aggregates)
 train_sentiment_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(aggregates)
 test_sentiment_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()<merge>
rf = RandomForestClassifier(random_state = 10)
 cv = cross_validate(rf,x_train,y_train,cv=5, return_train_score = True, return_estimator = True)
 classifiers['RandomForestNotScaled'] = cv
print('Average performance on training set:
')
 print(cv['train_score'].mean())
 print('
Average performance on test set:
')
 print(cv['test_score'].mean() )
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]<merge>
rf = RandomForestClassifier(random_state = 10)
 cv = cross_validate(rf,x_train_scaled,y_train,cv=5, return_train_score = True, return_estimator = True)
 classifiers['RandomForestScaled'] = cv
print('Average performance on training set:
')
 print(cv['train_score'].mean())
 print('
Average performance on test set:
')
 print(cv['test_score'].mean() )
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )<merge>
svc = SVC(probability = True)
 cv = cross_validate(svc, x_train, y_train,cv=5, return_train_score = True, return_estimator = True)
 classifiers['SVCNotScaled'] = cv
print('Average performance on training set:
')
 print(cv['train_score'].mean())
 print('
Average performance on test set:
')
 print(cv['test_score'].mean() )
train_proc = train_proc.merge(breed_score_df,how='left',left_on='main_breed_BreedName',right_on='breed_name')
 test_proc = test_proc.merge(breed_score_df,how='left',left_on='main_breed_BreedName',right_on='breed_name' )<concatenate>
svc = SVC(probability = True)
 cv = cross_validate(svc,x_train_scaled,y_train,cv=5, return_train_score = True, return_estimator = True)
 classifiers['SVCScaled'] = cv
print('Average performance on training set:
')
 print(cv['train_score'].mean())
 print('
Average performance on test set:
')
 print(cv['test_score'].mean() )
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False)
 print('NaN structure:
{}'.format(np.sum(pd.isnull(X))))<define_variables>
for i in classifiers:
print(i + ": "+"train: "+str(classifiers.get(i)['train_score'].mean())+" - test: " + str(classifiers.get(i)['test_score'].mean()), end='
' )