Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<define_variables>
%matplotlib inline
np.random.seed(seed=1337)
 warnings.filterwarnings('ignore')
 tqdm.pandas(tqdm_notebook)
 split_char = '/'
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print(f'num of train images files: {len(train_image_files)}')
 print(f'num of train metadata files: {len(train_metadata_files)}')
 print(f'num of train sentiment files: {len(train_sentiment_files)}')
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print(f'num of test images files: {len(test_image_files)}')
 print(f'num of test metadata files: {len(test_metadata_files)}')
 print(f'num of test sentiment files: {len(test_sentiment_files)}' )<load_pretrained>
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv' )
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_json_file(self, filename):
with open(filename, 'r', encoding='utf-8')as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns')
 file_sentences_sentiment_df = pd.DataFrame(
 {
'magnitude_sum': file_sentences_sentiment['magnitude'].sum(axis=0),
'score_sum': file_sentences_sentiment['score'].sum(axis=0),
'magnitude_mean': file_sentences_sentiment['magnitude'].mean(axis=0),
'score_mean': file_sentences_sentiment['score'].mean(axis=0),
'magnitude_var': file_sentences_sentiment['magnitude'].var(axis=0),
'score_var': file_sentences_sentiment['score'].var(axis=0),
}, index=[0] 
)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
df_sentiment = pd.concat([df_sentiment, file_sentences_sentiment_df], axis=1)
 df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = f'.. /input/petfinder-adoption-prediction/{mode}_sentiment/{pet_id}.json'
try:
sentiment_file = pet_parser.open_json_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob(f'.. /input/petfinder-adoption-prediction/{mode}_metadata/{pet_id}*.json'))
if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_json_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()<categorify>
debug = False
if debug:
train = train[:500]
test = test[:200]
aggregates = ['sum', 'mean', 'var']
sent_agg = ['sum']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 train_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 test_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()<merge>
import cv2
import os
from keras.applications.densenet import preprocess_input, DenseNet121
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]<merge>
img_size = 256
batch_size = 256
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )<concatenate>
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights=".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False )<define_variables>
pet_ids = train['PetID'].values
n_batches = int(np.ceil(len(pet_ids)/ batch_size))
features = {}
for b in tqdm_notebook(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID']<merge>
train_feats = pd.DataFrame.from_dict(features, orient='index')
 train_feats.columns = [f'pic_{i}' for i in range(train_feats.shape[1])]
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID' )<feature_engineering>
pet_ids = test['PetID'].values
n_batches = int(np.ceil(len(pet_ids)/ batch_size))
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/test_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]<feature_engineering>
test_feats = pd.DataFrame.from_dict(features, orient='index')
 test_feats.columns = [f'pic_{i}' for i in range(test_feats.shape[1])]
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('none' )<categorify>
train_feats = train_feats.reset_index()
train_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 test_feats = test_feats.reset_index()
test_feats.rename({'index': 'PetID'}, axis='columns', inplace=True )
X_temp['Length_Description'] = X_text['Description'].map(len)
 X_temp['Length_metadata_annots_top_desc'] = X_text['metadata_annots_top_desc'].map(len)
 X_temp['Lengths_sentiment_entities'] = X_text['sentiment_entities'].map(len )<feature_engineering>
all_ids = pd.concat([train, test], axis=0, ignore_index=True, sort=False)[['PetID']]
all_ids.shape
n_components = 16
text_features = []
for i in X_text.columns:
print(f'generating features from: {i}')
 tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_{}_'.format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )<merge>
n_components = 32
svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 features_df = pd.concat([train_feats, test_feats], axis=0)
 features = features_df[[f'pic_{i}' for i in range(256)]].values
svd_col = svd_.fit_transform(features)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('IMG_SVD_')
 img_features = pd.concat([all_ids, svd_col], axis=1 )
X_temp = X_temp.merge(img_features, how='left', on='PetID' )<define_variables>
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )
train_df_ids = train[['PetID']]
test_df_ids = test[['PetID']]
train_df_imgs = pd.DataFrame(train_image_files)
 train_df_imgs.columns = ['image_filename']
train_imgs_pets = train_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 def getSize(filename):
st = os.stat(filename)
 return st.st_size
def getDimensions(filename):
img_size = Image.open(filename ).size
return img_size
train_df_imgs['image_size'] = train_df_imgs['image_filename'].apply(getSize)
 train_df_imgs['temp_size'] = train_df_imgs['image_filename'].apply(getDimensions)
 train_df_imgs['width'] = train_df_imgs['temp_size'].apply(lambda x : x[0])
 train_df_imgs['height'] = train_df_imgs['temp_size'].apply(lambda x : x[1])
 train_df_imgs = train_df_imgs.drop(['temp_size'], axis=1)
 test_df_imgs['image_size'] = test_df_imgs['image_filename'].apply(getSize)
 test_df_imgs['temp_size'] = test_df_imgs['image_filename'].apply(getDimensions)
 test_df_imgs['width'] = test_df_imgs['temp_size'].apply(lambda x : x[0])
 test_df_imgs['height'] = test_df_imgs['temp_size'].apply(lambda x : x[1])
 test_df_imgs = test_df_imgs.drop(['temp_size'], axis=1)
 aggs = {
'image_size': ['sum', 'mean', 'var'],
'width': ['sum', 'mean', 'var'],
'height': ['sum', 'mean', 'var'],
}
agg_train_imgs = train_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()
agg_test_imgs = test_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index()
agg_imgs = pd.concat([agg_train_imgs, agg_test_imgs], axis=0 ).reset_index(drop=True )<merge>
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
if debug:
train_image_files = train_image_files[:1000]
train_metadata_files = train_metadata_files[:1000]
train_sentiment_files = train_sentiment_files[:1000]
print(f'num of train images files: {len(train_image_files)}')
 print(f'num of train metadata files: {len(train_metadata_files)}')
 print(f'num of train sentiment files: {len(train_sentiment_files)}')
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print(f'num of test images files: {len(test_image_files)}')
 print(f'num of test metadata files: {len(test_metadata_files)}')
 print(f'num of test sentiment files: {len(test_sentiment_files)}' )
X_temp = X_temp.merge(agg_imgs, how='left', on='PetID' )<drop_column>
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_json_file(self, filename):
with open(filename, 'r', encoding='utf-8')as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns')
 file_sentences_sentiment_df = pd.DataFrame(
 {
'magnitude_sum': file_sentences_sentiment['magnitude'].sum(axis=0),
'score_sum': file_sentences_sentiment['score'].sum(axis=0),
'magnitude_mean': file_sentences_sentiment['magnitude'].mean(axis=0),
'score_mean': file_sentences_sentiment['score'].mean(axis=0),
'magnitude_var': file_sentences_sentiment['magnitude'].var(axis=0),
'score_var': file_sentences_sentiment['score'].var(axis=0),
}, index=[0] 
)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
df_sentiment = pd.concat([df_sentiment, file_sentences_sentiment_df], axis=1)
 df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = f'.. /input/petfinder-adoption-prediction/{mode}_sentiment/{pet_id}.json'
try:
sentiment_file = pet_parser.open_json_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
for ind in range(1,200):
metadata_filename = '.. /input/petfinder-adoption-prediction/{}_metadata/{}-{}.json'.format(mode, pet_id, ind)
 try:
metadata_file = pet_parser.open_json_file(metadata_filename)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 except FileNotFoundError:
break
if dfs_metadata:
dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()
X_temp = X_temp.drop(to_drop_columns, axis=1 )<define_variables>
train_pet_ids = train.PetID.unique()
test_pet_ids = test.PetID.unique()
dfs_train = Parallel(n_jobs=-1, verbose=1 )(
 delayed(extract_additional_features )(i, mode='train')for i in train_pet_ids)
 train_dfs_sentiment = [x[0] for x in dfs_train if isinstance(x[0], pd.DataFrame)]
train_dfs_metadata = [x[1] for x in dfs_train if isinstance(x[1], pd.DataFrame)]
train_dfs_sentiment = pd.concat(train_dfs_sentiment, ignore_index=True, sort=False)
 train_dfs_metadata = pd.concat(train_dfs_metadata, ignore_index=True, sort=False)
 print(train_dfs_sentiment.shape, train_dfs_metadata.shape)
 dfs_test = Parallel(n_jobs=-1, verbose=1 )(
 delayed(extract_additional_features )(i, mode='test')for i in test_pet_ids)
 test_dfs_sentiment = [x[0] for x in dfs_test if isinstance(x[0], pd.DataFrame)]
test_dfs_metadata = [x[1] for x in dfs_test if isinstance(x[1], pd.DataFrame)]
test_dfs_sentiment = pd.concat(test_dfs_sentiment, ignore_index=True, sort=False)
 test_dfs_metadata = pd.concat(test_dfs_metadata, ignore_index=True, sort=False)
 print(test_dfs_sentiment.shape, test_dfs_metadata.shape )
X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1 )<count_missing_values>
aggregates = ['sum', 'mean', 'var']
sent_agg = ['sum']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 train_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 test_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()
X_train_non_null.isnull().any().any() , X_test_non_null.isnull().any().any()<import_modules>
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<train_on_grid>
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']<init_hyperparams>
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False )
xgb_params = {
'eval_metric': 'rmse',
'seed': 1500,
'learning_rate': 0.01,
'subsample': 1,
'gamma':5,
'colsample_bytree': 0.85,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
}<prepare_x_and_y>
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID']
def run_xgb(params, X_train, X_test):
n_splits = 10
verbose_eval = 1000
num_rounds = 60000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test<train_model>
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID' )
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null )<compute_train_metric>
for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk )<predict_on_test>
X_temp = X_temp.merge(img_features, how='left', on='PetID' )
coefficients_ = coefficients.copy()
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}' )<count_values>
len(np.asarray(np.array([[1,2,3],[4,5,6]])).shape )
Counter(train_predictions )<count_values>
replace_backslash = lambda x: x.replace('\', '/')
 vectorized_replace_backslash = np.vectorize(replace_backslash)
 aggs = {
'image_blurness': ['sum', 'mean', 'var'],
'image_size': ['sum', 'mean', 'var'],
'image_width': ['sum', 'mean', 'var'],
'image_height': ['sum', 'mean', 'var'],
}
Counter(test_predictions )<save_to_csv>
def get_image_features(img_info):
img_info["image_size"] = getSize(img_info['image_filename'])
 try:
im = Image.open(img_info['image_filename'])
 except Exception as e:
im = np.NaN
if im != np.NaN:
dimension = getDimensions(im)
 img_info["image_width"] = dimension[0]
img_info["image_height"] = dimension[1]
img_info["image_blurness"] = get_blurrness_score(im)
 else:
img_info["image_width"] = np.NaN
img_info["image_height"] = np.NaN
img_info["image_blurness"] = np.NaN
return img_info
submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions})
 submission.to_csv('submission.csv', index=False )<load_from_csv>
train_df_ids = train[['PetID']]
train_image_files = vectorized_replace_backslash(train_image_files)
 train_df_imgs = pd.DataFrame(train_image_files)
 train_df_imgs.columns = ['image_filename']
train_imgs_pets = train_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 train_image_features = train_df_imgs.assign(PetID=train_imgs_pets)
 with timer("getting image extra image features for train: "):
train_image_features = train_image_features.apply(get_image_features, axis=1 )
%matplotlib inline
np.random.seed(seed=1337)
 warnings.filterwarnings('ignore')
 split_char = '/'
os.listdir('.. /input')
 train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv' )<create_dataframe>
agg_train_imgs = train_image_features.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()
train_feats = pd.DataFrame.from_dict(features, orient='index')
 train_feats.columns = [f'pic_{i}' for i in range(train_feats.shape[1])]<define_variables>
test_df_ids = test[['PetID']]
test_image_files = vectorized_replace_backslash(test_image_files)
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 test_image_features = test_df_imgs.assign(PetID=test_imgs_pets)
 with timer("getting image extra image features for test: "):
test_image_features = test_image_features.apply(get_image_features, axis=1 )
pet_ids = test['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/test_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
test_feats = pd.DataFrame.from_dict(features, orient='index')
 test_feats.columns = [f'pic_{i}' for i in range(test_feats.shape[1])]
train_feats = train_feats.reset_index()
train_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 test_feats = test_feats.reset_index()
test_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 all_ids = pd.concat([train, test], axis=0, ignore_index=True, sort=False)[['PetID']]
all_ids.shape<create_dataframe>
agg_test_imgs = test_image_features.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index()
n_components = 16
svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 features_df = pd.concat([train_feats, test_feats], axis=0)
 features = features_df[[f'pic_{i}' for i in range(256)]].values
svd_col = svd_.fit_transform(features)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('IMG_SVD_')
 features = features_df[[f'pic_{i}' for i in range(256)]]
img_features = pd.concat([all_ids, svd_col], axis=1)
 gc.collect()<load_from_csv>
agg_imgs = pd.concat([agg_train_imgs, agg_test_imgs], axis=0 ).reset_index(drop=True)
 agg_imgs_columns = list(agg_imgs.columns)
 agg_imgs_columns.remove('PetID' )
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv')
 train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print(f'num of train images files: {len(train_image_files)}')
 print(f'num of train metadata files: {len(train_metadata_files)}')
 print(f'num of train sentiment files: {len(train_sentiment_files)}')
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print(f'num of test images files: {len(test_image_files)}')
 print(f'num of test metadata files: {len(test_metadata_files)}')
 print(f'num of test sentiment files: {len(test_sentiment_files)}' )<load_pretrained>
X_temp = X_temp.merge(agg_imgs, how='left', on='PetID' )
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_json_file(self, filename):
with open(filename, 'r', encoding='utf-8')as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns')
 file_sentences_sentiment_df = pd.DataFrame(
 {
'magnitude_sum': file_sentences_sentiment['magnitude'].sum(axis=0),
'score_sum': file_sentences_sentiment['score'].sum(axis=0),
'magnitude_mean': file_sentences_sentiment['magnitude'].mean(axis=0),
'score_mean': file_sentences_sentiment['score'].mean(axis=0),
'magnitude_var': file_sentences_sentiment['magnitude'].var(axis=0),
'score_var': file_sentences_sentiment['score'].var(axis=0),
}, index=[0] 
)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
df_sentiment = pd.concat([df_sentiment, file_sentences_sentiment_df], axis=1)
 df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = f'.. /input/petfinder-adoption-prediction/{mode}_sentiment/{pet_id}.json'
try:
sentiment_file = pet_parser.open_json_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob(f'.. /input/petfinder-adoption-prediction/{mode}_metadata/{pet_id}*.json'))
if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_json_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()<feature_engineering>
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('none' )
aggregates = ['sum', 'mean', 'var', 'nunique', 'max']
sent_agg = ['sum']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 train_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 test_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()<merge>
n_components = 32
text_features = []
for i in X_text.columns:
print(f'generating features from: {i}')
 tfv = TfidfVectorizer(max_features=None, strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_{}_'.format(i))
current_text = X_temp[i].str.lower().astype(str)
 X_temp[i] = X_temp[i].astype(str)
 X_temp[i + '_num_chars'] = current_text.apply(len)
 X_temp[i + '_num_words'] = current_text.apply(lambda comment: len(comment.split()))
X_temp[i + '_num_unique_words'] = current_text.apply(lambda comment: len(set(w for w in comment.split())))
 X_temp[i + '_words_vs_unique'] = X_temp[i + '_num_unique_words'] / X_temp[i+'_num_words'] * 100
X_temp[i + '_count_letters'] = current_text.apply(lambda x: len(str(x)))
 X_temp[i + "_count_punctuations"] = current_text.apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
X_temp[i + "_mean_word_len"] = current_text.apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))
text_features.append(svd_col)
 X_temp['words_vs_unique_description'] = X_temp['Description_num_unique_words'] / X_temp['Description_num_words'] * 100
X_temp['words_vs_unique_description'] = X_temp['Description_num_unique_words'] / X_temp['Description_num_words'] * 100
X_temp['len_description'] = X_temp['Description'].apply(lambda x: len(x))
X_temp['average_description_word_length'] = X_temp['len_description'] / X_temp['Description_num_words']
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]<merge>
embeddings_index = {}
time_start_load_embedding = time.time()
with open('.. /input/fatsttext-common-crawl/crawl-300d-2M/crawl-300d-2M.vec', encoding='utf8')as f:
for line in f:
values = line.rstrip().rsplit(' ')
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
print("finished loading embeddings file in: ", time.time() - time_start_load_embedding )
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )<concatenate>
def sum_embeddings_vector(x):
sum_bag_of_words = np.zeros(300)
 words = x.split(' ')
 count = 0
for w in words:
if w in embeddings_index:
sum_bag_of_words += embeddings_index.get(w)
 count += 1
if count > 0:
sum_bag_of_words /= count
return pd.Series(sum_bag_of_words )
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False )<define_variables>
svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 for i in X_text.columns:
emb_feat = X_text[i].apply(sum_embeddings_vector)
 emb_feat = svd_.fit_transform(emb_feat)
 emb_feat = pd.DataFrame(emb_feat)
 emb_feat = emb_feat.add_prefix('word_embedding_' + i + '_')
 text_features.append(emb_feat )
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID']<define_variables>
text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1 )
def is_popular_name(name):
most_popular_names = ["Max", "Sam", "Lady", "Bear", "Smokey", "Shadow", "Kitty", "Molly", "Buddy", "Brandy", "Ginger", "Baby", "Misty", "Missy", "Pepper", "Jake", "Bandit", "Tiger", "Samantha", "Lucky", "Muffin", "Princess", "Maggie", "Charlie", "Sheba", "Rocky", "Patches", "Tigger", "Rusty", "Buster", "Lucy", "Luna", "Sadie", "Sophie", "Jack"]
return int(name in most_popular_names )<merge>
for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )
X_temp['isPopularName'] = X_temp['Name'].apply(is_popular_name)
 rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID')
 for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('none')
 def set_pet_breed(b1, b2):
if(b1 in(0, 307)) &(b2 in(0, 307)) :
return 4
elif(b1 == 307)&(b2 not in(0, 307)) :
return 3
elif(b2 == 307)&(b1 not in(0, 307)) :
return 3
elif(b1 not in(0, 307)) &(b2 not in(0, 307)) &(b1 != b2):
return 2
elif(b1 == 0)&(b2 not in(0, 307)) :
return 1
elif(b2 == 0)&(b1 not in(0, 307)) :
return 1
elif(b1 not in(0, 307)) &(b2 not in(0, 307)) &(b1 == b2):
return 0
else:
return 3
X_temp["Pet_Breed"] = X_temp.apply(lambda x: set_pet_breed(x['Breed1'], x['Breed2']), axis=1 )<string_transform>
rescuer_ids = X_temp['RescuerID'].values
def avg_sen_len(sent_text):
sum_len = 0
for sent in sent_text:
tokens = word_tokenize(sent)
 sum_len += len(tokens)
 return sum_len/len(sent_text)
 def amount_of_words(text):
tp = type(text)is str
if not tp:
return -1
bag_of_words = nltk.word_tokenize(text)
 return len(bag_of_words)
 def num_senten(text):
tp = type(text)is str
if not tp:
return -1
sent_text = nltk.sent_tokenize(text)
 return len(sent_text )<categorify>
X_temp = X_temp.drop(to_drop_columns, axis=1 )
X_temp['num_senten'] = X_text['Description'].apply(num_senten)
 X_temp['amount_of_words'] = X_text['Description'].apply(amount_of_words)
 X_temp['Length_Description'] = X_text['Description'].map(len)
 X_temp['Length_metadata_annots_top_desc'] = X_text['metadata_annots_top_desc'].map(len)
 X_temp['Lengths_sentiment_entities'] = X_text['sentiment_entities'].map(len )<categorify>
X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1 )
<feature_engineering>
X_train_non_null.isnull().any().any() , X_test_non_null.isnull().any().any()
n_components = 16
text_features = []
for i in X_text.columns:
print(f'generating features from: {i}')
 tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_{}_'.format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )<merge>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )
X_temp = X_temp.merge(img_features, how='left', on='PetID' )<define_variables>
def to_bins(x, borders):
for i in range(len(borders)) :
if x <= borders[i]:
return i
return len(borders)
 class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _loss(self, coef, X, y, idx):
X_p = np.array([to_bins(pred, coef)for pred in X])
 ll = -quadratic_weighted_kappa(y, X_p)
 return ll
def fit(self, X, y):
coef = [1.5, 2.0, 2.5, 3.0]
golden1 = 0.618
golden2 = 1 - golden1
ab_start = [(1, 2),(1.5, 2.5),(2, 3),(2.5, 3.5)]
for it1 in range(10):
for idx in range(4):
a, b = ab_start[idx]
coef[idx] = a
la = self._loss(coef, X, y, idx)
 coef[idx] = b
lb = self._loss(coef, X, y, idx)
 for it in range(20):
if la > lb:
a = b -(b - a)* golden1
coef[idx] = a
la = self._loss(coef, X, y, idx)
 else:
b = b -(b - a)* golden2
coef[idx] = b
lb = self._loss(coef, X, y, idx)
 self.coef_ = {'x': coef}
def predict(self, X, coef):
X_p = np.array([to_bins(pred, coef)for pred in X])
 return X_p
def coefficients(self):
return self.coef_['x']
train_df_ids = train[['PetID']]
test_df_ids = test[['PetID']]
train_df_imgs = pd.DataFrame(train_image_files)
 train_df_imgs.columns = ['image_filename']
train_imgs_pets = train_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 def getSize(filename):
st = os.stat(filename)
 return st.st_size
def getDimensions(filename):
img_size = Image.open(filename ).size
return img_size
train_df_imgs['image_size'] = train_df_imgs['image_filename'].apply(getSize)
 train_df_imgs['temp_size'] = train_df_imgs['image_filename'].apply(getDimensions)
 train_df_imgs['width'] = train_df_imgs['temp_size'].apply(lambda x : x[0])
 train_df_imgs['height'] = train_df_imgs['temp_size'].apply(lambda x : x[1])
 train_df_imgs = train_df_imgs.drop(['temp_size'], axis=1)
 test_df_imgs['image_size'] = test_df_imgs['image_filename'].apply(getSize)
 test_df_imgs['temp_size'] = test_df_imgs['image_filename'].apply(getDimensions)
 test_df_imgs['width'] = test_df_imgs['temp_size'].apply(lambda x : x[0])
 test_df_imgs['height'] = test_df_imgs['temp_size'].apply(lambda x : x[1])
 test_df_imgs = test_df_imgs.drop(['temp_size'], axis=1)
 aggs = {
'image_size': ['sum', 'mean', 'var'],
'width': ['sum', 'mean', 'var'],
'height': ['sum', 'mean', 'var'],
}
agg_train_imgs = train_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()
agg_test_imgs = test_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index()
agg_imgs = pd.concat([agg_train_imgs, agg_test_imgs], axis=0 ).reset_index(drop=True )<merge>
import lightgbm as lgb
import xgboost as xgb
from sklearn.model_selection import StratifiedKFold, GroupKFold
X_temp = X_temp.merge(agg_imgs, how='left', on='PetID' )<define_variables>
ntrain = X_train_non_null.shape[0]
ntest = X_test_non_null.shape[0]
NFOLDS = 10
X_train_non_null = X_train.fillna(-1000)
 X_test_non_null = X_test.fillna(-1000 )<count_missing_values>
def stratified_group_k_fold(X, y, groups, k, seed=None):
labels_num = np.max(y)+ 1
y_counts_per_group = defaultdict(lambda: np.zeros(labels_num))
y_distr = Counter()
for label, g in zip(y, groups):
y_counts_per_group[g][label] += 1
y_distr[label] += 1
y_counts_per_fold = defaultdict(lambda: np.zeros(labels_num))
groups_per_fold = defaultdict(set)
 def eval_y_counts_per_fold(y_counts, fold):
y_counts_per_fold[fold] += y_counts
std_per_label = []
for label in range(labels_num):
label_std = np.std([y_counts_per_fold[i][label] / y_distr[label] for i in range(k)])
 std_per_label.append(label_std)
 y_counts_per_fold[fold] -= y_counts
return np.mean(std_per_label)
 groups_and_y_counts = list(y_counts_per_group.items())
 random.Random(seed ).shuffle(groups_and_y_counts)
 for g, y_counts in sorted(groups_and_y_counts, key=lambda x: -np.std(x[1])) :
best_fold = None
min_eval = None
for i in range(k):
fold_eval = eval_y_counts_per_fold(y_counts, i)
 if min_eval is None or fold_eval < min_eval:
min_eval = fold_eval
best_fold = i
y_counts_per_fold[best_fold] += y_counts
groups_per_fold[best_fold].add(g)
 all_groups = set(groups)
 for i in range(k):
train_groups = all_groups - groups_per_fold[i]
test_groups = groups_per_fold[i]
train_indices = [i for i, g in enumerate(groups)if g in train_groups]
test_indices = [i for i, g in enumerate(groups)if g in test_groups]
yield train_indices, test_indices
X_train_non_null.isnull().any().any() , X_test_non_null.isnull().any().any()<import_modules>
def get_oof(clf, X, y, X_test, groups):
oof_train = np.zeros(( ntrain,))
oof_test = np.zeros(( ntest,))
oof_test_skf = np.empty(( NFOLDS, ntest))
for i,(train_index, test_index)in enumerate(stratified_group_k_fold(X, y, rescuer_ids, NFOLDS, 1337)) :
print('Training for fold: ', i + 1)
 x_tr = X.iloc[train_index, :]
y_tr = y[train_index]
x_te = X.iloc[test_index, :]
y_te = y[test_index]
clf.train(x_tr, y_tr, x_val=x_te, y_val=y_te)
 oof_train[test_index] = clf.predict(x_te)
 oof_test_skf[i, :] = clf.predict(X_test)
 oof_test[:] = oof_test_skf.mean(axis=0)
 return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1 )
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<train_on_grid>
class SklearnWrapper(object):
def __init__(self, clf, seed=0, params=None):
params['random_state'] = seed
self.clf = clf(**params)
 def train(self, x_train, y_train, **kwargs):
self.clf.fit(x_train, y_train)
 def predict(self, x):
return self.clf.predict(x)
 class XgbWrapper(object):
def __init__(self, params=None):
self.param = params
self.nrounds = params.pop('nrounds', 60000)
 self.early_stop_rounds = params.pop('early_stop_rounds', 2000)
 def train(self, x_train, y_train, **kwargs):
dtrain = xgb.DMatrix(x_train, label=y_train)
 dvalid = xgb.DMatrix(data=kwargs['x_val'], label=kwargs['y_val'])
 watchlist = [(dtrain, 'train'),(dvalid, 'valid')]
self.model = xgb.train(dtrain=dtrain, num_boost_round=self.nrounds, evals=watchlist, early_stopping_rounds=self.early_stop_rounds,
verbose_eval=1000, params=self.param)
 def predict(self, x):
return self.model.predict(xgb.DMatrix(x), ntree_limit=self.model.best_ntree_limit)
 class LGBWrapper(object):
def __init__(self, params=None):
self.param = params
self.num_rounds = params.pop('nrounds', 60000)
 self.early_stop_rounds = params.pop('early_stop_rounds', 2000)
 def train(self, x_train, y_train, **kwargs):
dtrain = lgb.Dataset(x_train, label=y_train)
 dvalid = lgb.Dataset(kwargs['x_val'], label=kwargs['y_val'])
 watchlist = [dtrain, dvalid]
print('training LightGBM with params: ', self.param)
 self.model = lgb.train(
 self.param,
train_set=dtrain,
num_boost_round=self.num_rounds,
valid_sets=watchlist,
verbose_eval=1000,
early_stopping_rounds=self.early_stop_rounds 
)
 def predict(self, x):
return self.model.predict(x, num_iteration=self.model.best_iteration )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']<init_hyperparams>
lgbm_params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': 'rmse',
'nrounds': 50000,
'early_stop_rounds': 2000,
'max_depth': 4,
'num_leaves': 46,
'feature_fraction': 0.6,
'bagging_fraction': 0.9,
'bagging_freq': 8,
'learning_rate': 0.019,
'verbose': 0
}
lgb_wrapper = LGBWrapper(lgbm_params)
 with timer('Training LightGBM'):
lgb_oof_train, lgb_oof_test = get_oof(lgb_wrapper, X_train_non_null.drop(['AdoptionSpeed'], axis=1), X_train_non_null['AdoptionSpeed'].values.astype(int), X_test_non_null, groups=rescuer_ids )
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'eta': 0.0123,
'subsample': 0.8,
'colsample_bytree': 0.85,
'tree_method': 'gpu_hist',
'device': 'gpu',
'max_depth':6,
'silent': 1,
} 
<train_model>
rf_params = {
'n_jobs': -1,
'n_estimators': 150,
'max_features': 'auto',
'max_depth': 20,
'min_samples_leaf': 2,
}
rf = SklearnWrapper(clf=RandomForestRegressor, seed=1337, params=rf_params)
 with timer('Training Random Forest'):
rf_oof_train, rf_oof_test = get_oof(rf, X_train_non_null.drop(['AdoptionSpeed'], axis=1), X_train_non_null['AdoptionSpeed'].values.astype(int), X_test_non_null, groups=rescuer_ids )
print('MODEL LEARNING STAGE' )<train_on_grid>
et_params = {
'n_jobs': -1,
'n_estimators': 150,
'max_features': 'auto',
'max_depth': 50,
'min_samples_leaf': 2,
}
et = SklearnWrapper(clf=ExtraTreesRegressor, seed=1337, params=et_params)
 with timer('Training Extra trees'):
et_oof_train, et_oof_test = get_oof(et, X_train_non_null.drop(['AdoptionSpeed'], axis=1), X_train_non_null['AdoptionSpeed'].values.astype(int), X_test_non_null, groups=rescuer_ids )
def run_xgb(params, X_train, X_test):
n_splits = 10
verbose_eval = 1000
num_rounds = 60000
early_stop = 500
oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
kf = GroupKFold(n_splits=n_splits)
 for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values, X_train['RescuerID']):
if 'RescuerID' in X_train.columns:
X_train = X_train.drop(to_drop_columns, axis=1)
 X_test = X_test.drop(to_drop_columns, axis=1)
 X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test<train_model>
xgb_params = {
'eval_metric': 'rmse',
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
'seed': 1337,
'nrounds': 60000,
'early_stop_rounds': 2000,
'eta': 0.025,
'subsample': 0.8,
'colsample_bytree': 0.6000000000000001,
'gamma': 0.65,
'max_depth': 4,
'min_child_weight': 5.0,
'n_estimators': 1000,
}
xgbWrapper = XgbWrapper(xgb_params)
 with timer('Training Xgboost'):
xgb_oof_train, xgb_oof_test = get_oof(xgbWrapper, X_train_non_null.drop(['AdoptionSpeed'], axis=1), X_train_non_null['AdoptionSpeed'].values.astype(int), X_test_non_null, groups=rescuer_ids )
%%time
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null )<compute_train_metric>
x_train_ensemble = np.concatenate(( et_oof_train, rf_oof_train, lgb_oof_train, xgb_oof_train), axis=1)
 x_test_ensemble = np.concatenate(( et_oof_test, rf_oof_test, lgb_oof_test, xgb_oof_test), axis=1 )
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk )<predict_on_test>
ridge_params = {
'alpha':50.0,
'fit_intercept':True,
'normalize':False,
'copy_X':True,
'max_iter':None,
'tol':0.001,
'solver':'auto',
'random_state':1337
}
ridge = SklearnWrapper(clf=Ridge, seed=1337, params=ridge_params)
 final_oof_train = np.zeros(( ntrain,))
final_oof_test = np.zeros(( ntest,))
final_oof_test_skf = np.empty(( NFOLDS, ntest))
for i,(train_index, test_index)in enumerate(stratified_group_k_fold(x_train_ensemble, X_train_non_null['AdoptionSpeed'].values.astype(int), rescuer_ids, NFOLDS, 1337)) :
print('Training ridge regressor for fold: ', i + 1)
 x_tr = x_train_ensemble[train_index]
y_tr = X_train_non_null['AdoptionSpeed'].values[train_index]
x_te = x_train_ensemble[test_index]
y_te = X_train_non_null['AdoptionSpeed'].values[test_index]
print(x_tr.shape, y_tr.shape, x_te.shape, y_te.shape)
 ridge.train(x_tr, y_tr, x_val=x_te, y_val=y_te)
 final_oof_train[test_index] = ridge.predict(x_te)
 final_oof_test_skf[i, :] = ridge.predict(x_test_ensemble)
 final_oof_test[:] = final_oof_test_skf.mean(axis=0)
 final_oof_train = final_oof_train.reshape(-1, 1)
 final_oof_test = final_oof_test.reshape(-1, 1 )
coefficients_ = coefficients.copy()
coefficients_[0] = 1.66
coefficients_[1] = 2.13
coefficients_[3] = 2.85
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}' )<save_to_csv>
optR = OptimizedRounder()
optR.fit(final_oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(final_oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk )
print(Counter(train_predictions), Counter(test_predictions))
submission1 = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions})
 submission1.to_csv('submission1.csv', index=False )<drop_column>
gc.collect()
def run_xgb(params, X_train, X_test):
n_splits = 10
verbose_eval = 1000
num_rounds = 60000
early_stop = 500
if 'RescuerID' in X_train.columns:
X_train = X_train.drop(to_drop_columns, axis=1)
 X_test = X_test.drop(to_drop_columns, axis=1)
 if 'Pet_Breed' in X_train.columns:
X_train = X_train.drop(['Pet_Breed', 'RescuerID_COUNT'], axis=1)
 X_test = X_test.drop(['Pet_Breed', 'RescuerID_COUNT'], axis=1)
 kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test<train_model>
coefficients_ = coefficients.copy()
train_predictions = optR.predict(final_oof_train, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(final_oof_test, coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}' )
%%time
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null )<find_best_model_class>
Counter(train_predictions )
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk)
 coefficients_ = coefficients.copy()
coefficients_[0] = 1.66
coefficients_[1] = 2.13
coefficients_[3] = 2.85
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}')
 print(Counter(train_predictions), Counter(test_predictions))
submission2 = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions})
 submission2.to_csv('submission2.csv', index=False )<load_from_csv>
Counter(test_predictions )
<drop_column><EOS> 
submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions})
 submission.to_csv('submission.csv', index=False)
 submission.head()