Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<feature_engineering>
%matplotlib inline
np.random.seed(seed=1337)
 warnings.filterwarnings('ignore')
 split_char = '/'
class StemmedTfidfVectorizer(TfidfVectorizer):
def __init__(self, stemmer, *args, **kwargs):
super(StemmedTfidfVectorizer, self ).__init__(*args, **kwargs)
 self.stemmer = stemmer
def build_analyzer(self):
analyzer = super(StemmedTfidfVectorizer, self ).build_analyzer()
return lambda doc:(self.stemmer.stem(word)for word in analyzer(doc.replace('
', ' ')))
 def grouped_feature(X, gr_col, agg_cols, agg_funcs):
gr = X.groupby(gr_col)[agg_cols].agg(agg_funcs)
 gr.columns = [l1+'_'+'_'.join(gr_col)+'_'+l2 for l1,l2 in zip(gr.columns.get_level_values(0),
gr.columns.get_level_values(1)) ]
gr = gr.reset_index()
X = X.merge(gr, how='left', on=gr_col)
 return X[gr.columns.drop(gr_col)]
class StratifiedBagger(BaseEstimator,RegressorMixin):

 def __init__(self, clf=None, n_bags=10, random_state=None, labelize=False, early_stopping_rounds=None):
self.clf = clf
self.n_bags = n_bags
self.random_state = random_state if random_state is not None else SEED
self.labelize = labelize
self.early_stopping_rounds = early_stopping_rounds
def fit(self, X, y=None, **fit_params):
skf = StratifiedKFold(self.n_bags, shuffle=True, random_state=self.random_state)
 folds = skf.split(X, y)
 self.clf_ = []
self.y_ = y
for i,(train_idx,valid_idx)in enumerate(folds):
if isinstance(X, pd.DataFrame): 
(X_train,y_train,X_valid,y_valid)=(X.iloc[train_idx],y.iloc[train_idx],
X.iloc[valid_idx],y.iloc[valid_idx])
 else: 
(X_train,y_train,X_valid,y_valid)=(X[train_idx],y.iloc[train_idx],
X[valid_idx],y.iloc[valid_idx])
 clf = deepcopy(self.clf)
 self.clf_.append(clf)
 if self.early_stopping_rounds is None:
self.clf_[i].fit(X_train, y_train, **fit_params)
 else:
self.clf_[i].fit(X_train, y_train, eval_set=[(X_valid,y_valid)],
early_stopping_rounds=self.early_stopping_rounds,
verbose=False, **fit_params)
 if VERBOSE:
if isinstance(self.clf_[i], LGBMRegressor):
fig,ax = plt.subplots(1, 1, figsize=(12,12))
plot_importance(self.clf_[i], ax=ax)
 plt.show()
if isinstance(self.clf_[i], XGBRegressor):
fig,ax = plt.subplots(1, 1, figsize=(12,12))
xgb_plot_importance(self.clf_[i], ax=ax)
 plt.show()
if isinstance(self.clf_[i], CatBoostRegressor):
fea_imp = pd.DataFrame({'imp': self.clf_[i].feature_importances_,
'col': X_train.columns})
 fea_imp = fea_imp.sort_values(['imp', 'col'], ascending=[True, False])
 fea_imp.plot(kind='barh', x='col', y='imp', figsize=(12,12))
plt.show()
return self
def predict(self, X, y=None):
y_preds = np.zeros(( X.shape[0],self.n_bags))
for i in range(self.n_bags):
y_pred = minmax_scale(self.clf_[i].predict(X),(0,4))
if self.labelize:
y_pred = labelize_match_target(y_pred, self.y_)
 y_preds[:,i] = y_pred
if self.labelize:
y_preds = mode(y_preds.T)[0].T.flatten().astype(np.int32)
 else:
y_preds = y_preds.mean(-1)
 return y_preds
class OOFTransformer(TransformerMixin, BaseEstimator):

 def __init__(self, clf, n_folds=5, agg_func='mean', name=None):
self.clf = clf
self.n_folds = n_folds
self.agg_func = agg_func
self.name = name
if self.name is None:
self.name = str(self.clf.__class__ ).split('.')[-1][1:-2]
def fit_transform(self, X, y, **fit_kwargs):
self.name_ = 'oof_' + self.name
self.clf_ = []
skf = StratifiedKFold(self.n_folds, shuffle=True, random_state=SEED+1234)
 folds = skf.split(X, y)
 X_train = np.zeros(( X.shape[0],))
if not SUBMISSION and not USE_LB:
path = f'{self.name_}_{CV}_{FOLD}_X_train.csv'
if os.path.exists(path)and os.path.exists(path.replace('X_train','X_test')) :
if VERBOSE:
logger.info(f'Read existing for {self.name_}(fit)')
 return pd.read_csv(path)
 for i,(fold_idx,oof_idx)in enumerate(folds):
self.clf_.append(deepcopy(self.clf))
if VERBOSE:
logger.info(f'Generating {self.name_} oof {i+1}/{self.n_folds}')
 if isinstance(X, pd.DataFrame): 
(X_fold,y_fold,X_oof,y_oof)=(X.iloc[fold_idx],y.iloc[fold_idx],
X.iloc[oof_idx],y.iloc[oof_idx])
 else: 
(X_fold,y_fold,X_oof,y_oof)=(X[fold_idx],y.iloc[fold_idx],
X[oof_idx],y.iloc[oof_idx])
 self.clf_[i].fit(X_fold, y_fold)
 X_train[oof_idx] = self.clf_[i].predict(X_oof)
 if VERBOSE:
if isinstance(self.clf_[i], LGBMRegressor):
fig,ax = plt.subplots(1, 1, figsize=(18,18))
plot_importance(self.clf_[i], ax=ax)
 plt.show()
if isinstance(self.clf_[i], XGBRegressor):
fig,ax = plt.subplots(1, 1, figsize=(18,18))
xgb_plot_importance(self.clf_[i], ax=ax)
 plt.show()
if isinstance(self.clf_[i], CatBoostRegressor):
fea_imp = pd.DataFrame({'imp': self.clf_[i].feature_importances_,
'col': X_train.columns})
 fea_imp = fea_imp.sort_values(['imp', 'col'], ascending=[True, False] ).iloc[-30:]
fea_imp.plot(kind='barh', x='col', y='imp', figsize=(18,18))
plt.show()
X_train = pd.DataFrame(X_train, columns=[self.name_])
 if not SUBMISSION and not USE_LB:
X_train.to_csv(path, index=False)
 return X_train
def transform(self, X):
if not SUBMISSION and not USE_LB:
path = f'{self.name_}_{CV}_{FOLD}_X_test.csv'
if os.path.exists(path):
if VERBOSE:
print(f'Read existing for {self.name_}(transform)')
 return pd.read_csv(path)
 X_test = np.zeros(( X.shape[0],self.n_folds))
for i in range(self.n_folds):
X_test[:,i] = self.clf_[i].predict(X)
 if self.agg_func == 'mean':
X_test = X_test.mean(-1)
 elif self.agg_func == 'median':
X_test = np.median(X_test, axis=-1)
 else:
raise ValueError(f'Unknown aggregation {self.agg_func}')
 X_test = pd.DataFrame(X_test, columns=[self.name_])
 if not SUBMISSION and not USE_LB:
X_test.to_csv(path, index=False)
 return X_test
class PredictTransformer(BaseEstimator, TransformerMixin):
def __init__(self, clf, labelize=True, name=None):
self.clf = clf
self.labelize = labelize
self.name = name
if self.name is None:
self.name = str(self.clf.__class__ ).split('.')[-1][1:-2]
def fit(self, X, y, **fit_kwargs):
self.name_ = 'pred_' + self.name
self.clf.fit(X, y, **fit_kwargs)
 if VERBOSE:
print(f'Fitting {self.name_}...')
 if isinstance(self.clf, LGBMRegressor):
fig,ax = plt.subplots(1, 1, figsize=(18,18))
plot_importance(self.clf, ax=ax)
 plt.show()
if isinstance(self.clf, XGBRegressor):
fig,ax = plt.subplots(1, 1, figsize=(18,18))
xgb_plot_importance(self.clf, ax=ax)
 plt.show()
if isinstance(self.clf, CatBoostRegressor):
fea_imp = pd.DataFrame({'imp': self.clf.feature_importances_,
'col': X_train.columns})
 fea_imp = fea_imp.sort_values(['imp', 'col'], ascending=[True, False] ).iloc[-30:]
fea_imp.plot(kind='barh', x='col', y='imp', figsize=(18,18))
plt.show()
y_pred = self.clf.predict(X)
 if self.labelize:
self.y_ = y
return self
def transform(self, X):
y_pred = self.clf.predict(X)
 if self.labelize:
y_pred = labelize_match_target(y_pred, self.y_)
 return pd.DataFrame(y_pred, columns=[self.name_] )<load_from_csv>
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv' )
breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 color = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 state = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )<load_from_csv>
import cv2
import os
from keras.applications.densenet import preprocess_input, DenseNet121
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 logger.info('Train shape: %s' % str(train.shape))
logger.info('Test shape: %s' % str(test.shape))<merge>
img_size = 256
batch_size = 256
ext_state = pd.read_excel('.. /input/malaysian-state-data/external_state_data.xlsx',
decimal=',')[['State','Population2','2016 GDP']]
train = train.merge(ext_state, how='left', on='State')
 test = test.merge(ext_state, how='left', on='State' )<data_type_conversions>
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights=".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )
train['Description'] = train['Description'].fillna('' ).astype(str)
 test['Description'] = test['Description'].fillna('' ).astype(str )<define_variables>
pet_ids = train['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
def create_metadata(pet_id, train=True):
all_res = {}
for pic in range(1,2):
meta_path = '.. /input/petfinder-adoption-prediction/%s_metadata/%s-%s.json' %('train' if train else 'test', pet_id,pic)
 pic_path = '.. /input/petfinder-adoption-prediction/%s_images/%s-%s.jpg' %('train' if train else 'test', pet_id,pic)
 res = {
'PetID': pet_id,
f'vertex_y{pic}': 0,
f'label_score{pic}': 0,
}
try:
if os.path.exists(meta_path):
with open(meta_path, 'r')as f:
d = json.load(f)
 res[f'vertex_y{pic}'] = d['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
if 'labelAnnotations' in d:
res[f'label_score{pic}'] = d['labelAnnotations'][0]['score'] + 1
all_res.update(res)
 except:
logger.warning('Failed loading JSON files!')
 pass
return all_res<create_dataframe>
train_feats = pd.DataFrame.from_dict(features, orient='index')
 train_feats.columns = [f'pic_{i}' for i in range(train_feats.shape[1])]
logger.info('Creating metadata features...')
 train_metadata_feats = Parallel(n_jobs=4 )(delayed(create_metadata )(pet_id)
 for pet_id in tqdm_notebook(train['PetID'].unique()))
train_metadata_feats = pd.DataFrame(train_metadata_feats)
 create_metadata_test = partial(create_metadata, train=False)
 test_metadata_feats = Parallel(n_jobs=4 )(delayed(create_metadata_test )(pet_id)
 for pet_id in tqdm_notebook(test['PetID'].unique()))
test_metadata_feats = pd.DataFrame(test_metadata_feats)
 logger.info('Merging metadata features...')
 train = train.merge(train_metadata_feats, how='left', on='PetID')
 test = test.merge(test_metadata_feats, how='left', on='PetID')
 del train_metadata_feats,test_metadata_feats
gc.collect()
logger.info('Metadata features done!' )<merge>
pet_ids = test['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/test_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
logger.info('Creating image features...')
 train_img_feats,test_img_feats = create_features(normal, 'xception', 'avg')
 tmp_train,tmp_test = create_features(normal, 'nasnetlarge', 'avg')
 train_img_feats = train_img_feats.merge(tmp_train, how='left', on='PetID')
 test_img_feats = test_img_feats.merge(tmp_test, how='left', on='PetID')
 logger.info('Merging image features...')
 train = train.merge(train_img_feats, how='left', on='PetID')
 test = test.merge(test_img_feats, how='left', on='PetID')
 del train_img_feats,test_img_feats
gc.collect()
logger.info('Image features done!' )<data_type_conversions>
test_feats = pd.DataFrame.from_dict(features, orient='index')
 test_feats.columns = [f'pic_{i}' for i in range(test_feats.shape[1])]
def row_features(X):

 X['adopted'] = 1
X.loc[X['Description'].str.lower().str.contains('adopted'),'adopted'] = 0
X.loc[X['Description'].str.lower().str.contains('adoption'),'adopted'] = 2
rescuer_counts =(
 X[['RescuerID','PetID']] 
.groupby('RescuerID' ).agg({'PetID': lambda x:(x.count() > 1 ).astype(np.int8)})
.rename(columns={'PetID': 'MoreThanOneRescuerPet'})
.reset_index() 
)
 X = X.merge(rescuer_counts, how='left', on='RescuerID')
 return X[['adopted','MoreThanOneRescuerPet']]
class ExtractLeaky(TransformerMixin, BaseEstimator):

 def __init__(self, train, test):
self.train = train
self.test = test
def _ratio(self, gr1, gr2, agg_col='PetID', agg_func='count'):
req_cols = list(set(gr1 + gr2 + [agg_col]))
df = pd.concat([self.train[req_cols], self.test[req_cols]], axis=0)
 df1 = df.groupby(gr1)[agg_col].agg(agg_func ).to_frame('gr1' ).reset_index()
df2 = df.groupby(gr2)[agg_col].agg(agg_func ).to_frame('gr2' ).reset_index()
df = df.merge(df1, how='left', on=gr1 ).merge(df2, how='left', on=gr2)
 colname = '%s_%s_%s_%s' %(agg_col,agg_func,'-'.join(gr1),'-'.join(gr2))
df[colname] =(df['gr1'] / df['gr2'] ).replace([-np.inf,np.inf],np.nan)
 df = df.drop(['gr1','gr2'], axis=1)
 self.train_feats_ = pd.concat([
self.train_feats_, df[colname].iloc[:self.n_train_]], axis=1)
 self.test_feats_ = pd.concat([
self.test_feats_, df[colname].iloc[self.n_train_:].reset_index(drop=True)], axis=1)
 def fit_transform(self, X, y=None, **fit_kwargs):
self.n_train_ = self.train.shape[0]
self.train_feats_ = pd.DataFrame()
self.test_feats_ = pd.DataFrame()
self._ratio(['Breed1'], ['State'], 'PetID', 'count')
 self._ratio(['Breed1','RescuerID'], ['PetID'], 'PetID', 'count')
 self._ratio(['PetID'], ['Breed1'], 'PhotoAmt', 'mean')
 self._ratio(['RescuerID'], ['PetID'], 'Breed1', 'nunique')
 self.train_feats_['Breed1_nunique_RescuerID_PetID'] =(
 self.train_feats_['Breed1_nunique_RescuerID_PetID'].clip(1, self.test_feats_['Breed1_nunique_RescuerID_PetID'].max()))
self.test_feats_['Breed1_nunique_RescuerID_PetID'] =(
 self.test_feats_['Breed1_nunique_RescuerID_PetID'].clip(1, self.test_feats_['Breed1_nunique_RescuerID_PetID'].max()))
return self.train_feats_
def transform(self, X):
return self.test_feats_<concatenate>
train_feats = train_feats.reset_index()
train_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 test_feats = test_feats.reset_index()
test_feats.rename({'index': 'PetID'}, axis='columns', inplace=True )
target = 'AdoptionSpeed'
common_cols = test.columns.intersection(train.columns ).values.tolist()
train = train[common_cols + [target]]
test = test[common_cols]<feature_engineering>
all_ids = pd.concat([train, test], axis=0, ignore_index=True, sort=False)[['PetID']]
all_ids.shape
rescuer_ids = get_rescuer_ids(train )<split>
n_components = 32
svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 features_df = pd.concat([train_feats, test_feats], axis=0)
 features = features_df[[f'pic_{i}' for i in range(256)]].values
svd_col = svd_.fit_transform(features)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('IMG_SVD_')
 img_features = pd.concat([all_ids, svd_col], axis=1 )
if LB:
train_idx,test_idx = train_test_split(rescuer_ids['unique'],
stratify=rescuer_ids['count'], test_size=0.2, random_state=209321206+73)
 train_idx = [idx for sidx in train_idx for idx in sidx]
test_idx = [idx for sidx in test_idx for idx in sidx]
X_train,y_train,X_test,y_test =(train.iloc[train_idx].drop('AdoptionSpeed', axis=1),
train.iloc[train_idx][target],
train.iloc[test_idx].drop('AdoptionSpeed', axis=1),
train.iloc[test_idx][target])
 rescuer_ids = get_rescuer_ids(train.iloc[train_idx])
 else:
X_train,y_train,X_test = train.drop('AdoptionSpeed', axis=1),train[target],test<create_dataframe>
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )
def run_level0(X_train, y_train, X_test):
text_feats = PandasFeatureUnion([ 
('Original', PandasTransform(lambda x: x)) , 
('Raw description + TFIDF + SVD10', PandasPipeline([ 
('select', SelectColumns(['Description'])) , 
('preprocess', PandasTransform(lambda x: x.values.flatten().tolist())) , 
('tfidf', TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)) , 
('svd', TruncatedSVD(n_components=10, random_state=1337)) , 
('df', PandasTransform(lambda x:
pd.DataFrame(x, columns=[f'lvl0_text_raw_desc_tfidf_svd10_{i}' for i in range(10)])))
 ])) , 
('Stemmed description + TFIDF + SVD5', PandasPipeline([ 
('select', SelectColumns(['Description'])) , 
('preprocess', PandasTransform(lambda x: x.fillna('' ).values.flatten().tolist())) , 
('tfidf', StemmedTfidfVectorizer(SnowballStemmer('english', ignore_stopwords=False),
min_df=2, max_features=None, strip_accents='unicode', analyzer='word',
token_pattern=r'(?u)\b\w+\b', ngram_range=(1, 3), use_idf=1, smooth_idf=1,
sublinear_tf=1)) , 
('svd', TruncatedSVD(n_components=5, random_state=1337)) , 
('df', PandasTransform(lambda x:
pd.DataFrame(x, columns=[f'lvl0_text_stem_desc_tfidf_svd5_{i}' for i in range(5)])))
 ]))
], n_jobs=1)
 X_train = text_feats.fit_transform(X_train, y_train)
 X_test = text_feats.transform(X_test)
 img_feats = PandasFeatureUnion([ 
('Original', PandasTransform(lambda x: x)) , 
('NasNet + PCA5', PandasPipeline([ 
('select cols', SelectColumns(regex=r'(nasnetlarge_)')) , 
('impute', SimpleImputer(fill_value=0.)), 
('var_thres', VarianceThreshold(0.)), 
('pca', PCA(5, random_state=234)) , 
('df', PandasTransform(lambda x:
pd.DataFrame(x, columns=[f'lvl0_img_nasnet_pca5_{i}' for i in range(5)])))
 ])) , 
('NasNet + PCA400 + LR', OOFTransformer(make_pipeline(
 SelectColumns(regex=r'(nasnetlarge_)'),
SimpleImputer(fill_value=0.) ,
VarianceThreshold(0.) ,
PCA(400, random_state=234),
LinearRegression(
)
), name='lvl0_img_nasnet_pca400_lr')) , 
('Xception + SVD150 + ET', OOFTransformer(make_pipeline(
 SelectColumns(regex=r'(xception_)'),
SimpleImputer(fill_value=0.) ,
VarianceThreshold(0.) ,
TruncatedSVD(150, random_state=113120),
ExtraTreesRegressor(
 n_estimators=200,
random_state=10233 
)
), name='lvl0_img_xception_svd150_pca400_lr')) ,
], n_jobs=1)
 X_train = img_feats.fit_transform(X_train, y_train)
 X_test = img_feats.transform(X_test)
 if SUBMISSION or VERBOSE:
display(X_train.filter(like='oof_lvl0' ).corr())
 display(X_test.filter(like='oof_lvl0' ).corr())
 if SUBMISSION:
X_test.filter(like='oof_lvl0' ).to_csv('X_test_lvl0.csv', index=False)
 return X_train,X_test
def run_level1(X_train, y_train, X_test):
normal_feats = [
'Type', 'Age', 'Breed1', 'Breed2', 'Gender',
'Color1', 'Color2', 'Color3', 'MaturitySize',
'FurLength', 'Vaccinated', 'Dewormed', 'Sterilized',
'Health', 'Quantity', 'Fee', 'State', 'VideoAmt', 'PhotoAmt',
'Population2', '2016 GDP', 'adopted', 'MoreThanOneRescuerPet',
]
lvl1 = PandasFeatureUnion([ 
('Original', PandasTransform(lambda x: x)) , 
('LGB + Non-image', OOFTransformer(make_pipeline(
 SelectColumns(normal_feats),
LGBMRegressor(
 n_estimators=100,
num_leaves=16,
learning_rate=0.1,
max_depth=5,
colsample_bytree=0.5,
min_split_gain=0.02,
min_child_samples=100,
min_child_weight=0.02,
reg_lambda=0.01,
random_state=SEED+374,
objective='rmse' 
)
), name='lvl1_lgb_non_img')) , 
('LGB + Image and text', OOFTransformer(make_pipeline(
 SelectColumns(regex=r'(lvl0_text|lvl0_img|PhotoAmt|label_score|vertex_y)'),
LGBMRegressor(
 n_estimators=100,
num_leaves=16,
learning_rate=0.1,
max_depth=5,
colsample_bytree=0.5,
min_split_gain=0.02,
min_child_samples=100,
min_child_weight=0.02,
reg_lambda=0.01,
random_state=SEED+274,
objective='rmse' 
)
), name='lvl1_lgb_img_text')) , 
('LGB + All features bagging', OOFTransformer(make_pipeline(
 SelectColumns(regex=r'(lvl0|label_score|vertex_y|%s)' % '|'.join(normal_feats)) ,
BaggingRegressor(
 LGBMRegressor(
 n_estimators=100,
num_leaves=16,
learning_rate=0.1,
max_depth=5,
colsample_bytree=0.5,
min_split_gain=0.02,
min_child_samples=100,
min_child_weight=0.02,
reg_lambda=0.01,
random_state=SEED+743,
objective='rmse' 
),
n_estimators=20,
max_samples=0.8,
max_features=0.4,
bootstrap=True,
random_state=9843 
)
), name='lvl1_lgb_bagging'))
], n_jobs=1)
 X_train = lvl1.fit_transform(X_train, y_train)
 X_test = lvl1.transform(X_test)
 lvl1_rescuer_oof_mean = PandasFeatureUnion([ 
('original', PandasTransform(lambda x: x)) , 
('rescuermean', PandasTransform(lambda x:
grouped_feature(x, ['RescuerID'], ['oof_lvl1_lgb_bagging'], ['mean'])))
 ])
 X_train = lvl1_rescuer_oof_mean.fit_transform(X_train)
 X_test = lvl1_rescuer_oof_mean.transform(X_test)
 if SUBMISSION or VERBOSE:
display(X_train.filter(regex='(oof_lvl1_|pred_)' ).corr())
 display(X_test.filter(regex='(oof_lvl1_|pred_)' ).corr())
 if SUBMISSION:
X_test.filter(regex='(oof_lvl1_|pred_)' ).to_csv('X_test_lvl1.csv', index=False)
 return X_train,X_test
def run_level2(X_train, y_train, X_test):
normal_feats = [
'Type', 'Age', 'Breed1', 'Breed2', 'Gender',
'Color1', 'Color2', 'Color3', 'MaturitySize',
'FurLength', 'Vaccinated', 'Dewormed', 'Sterilized',
'Health', 'Quantity', 'Fee', 'State', 'VideoAmt', 'PhotoAmt',
'Population2', '2016 GDP', 'adopted', 'MoreThanOneRescuerPet',
]
lvl2 = PandasFeatureUnion([ 
('Best LGB', PredictTransformer(make_pipeline(
 SelectColumns(regex=r'(lvl0|lvl1|label_score|vertex_y|%s)' % '|'.join(normal_feats)) ,
LGBMRegressor(
 n_estimators=100,
num_leaves=16,
learning_rate=0.1,
max_depth=5,
colsample_bytree=0.5,
min_split_gain=0.02,
min_child_samples=100,
min_child_weight=0.02,
reg_lambda=0.01,
random_state=SEED+74,
objective='rmse' 
)
), name='lvl2_lgb')) , 
('Best CatBoost', PredictTransformer(make_pipeline(
 SelectColumns(regex=r'(lvl0|lvl1|label_score|vertex_y|%s)' % '|'.join(normal_feats)) ,
CatBoostRegressor(
 n_estimators=250,
learning_rate=0.1,
max_depth=6,
colsample_bylevel=0.5,
l2_leaf_reg=1.,
random_state=SEED+1932,
verbose=-1 
)
), name='lvl2_catboost')) , 
('Best Ridge', PredictTransformer(make_pipeline(
 SelectColumns(regex=r'(oof_lvl1_lgb)'),
DropColumns(regex=r'(RescuerID)'),
BaggingRegressor(
 BayesianRidge(
 normalize=False,
fit_intercept=True 
),
n_estimators=100,
max_samples=0.8,
max_features=1.0,
bootstrap=True,
random_state=93843 
)
), name='lvl2_ridge'))
], n_jobs=1)
 X_train = lvl2.fit_transform(X_train, y_train)
 X_test = lvl2.transform(X_test)
 if SUBMISSION or VERBOSE:
display(X_train.filter(regex='(oof_lvl2_|pred_)' ).corr())
 display(X_test.filter(regex='(oof_lvl2_|pred_)' ).corr())
 if SUBMISSION:
X_test.filter(regex='(oof_lvl2_|pred_)' ).to_csv('X_test_lvl2.csv', index=False)
 return X_train,X_test
def run_level3(X_train, y_train, X_test):
y_pred = mode(X_test.values.T)[0].T.flatten().astype(np.int32)
 return y_pred
def run_all(X_train, y_train, X_test):
X_train = X_train.reset_index(drop=True ).copy()
X_test = X_test.reset_index(drop=True ).copy()
if VERBOSE or SUBMISSION:
logger.info('>> Fitting basic pipeline...')
 pl = PandasPipeline([ 
('extract_features', PandasFeatureUnion([ 
('Original features', PandasTransform(lambda x: x)) , 
('Row features', PandasTransform(row_features)) , 
('Leaky features', ExtractLeaky(X_train, X_test))
])) , 
('Common values', FilterCommonValues(X_train, X_test, ['Breed1','Breed2','State'])) , 
('Categoricals', ConvertType([
'Type','Breed1','Breed2','Gender','Color1','Color2','Color3','MaturitySize',
'FurLength','Vaccinated','Dewormed','Sterilized','Health','State','adopted'],
'category'))
])
 X_train = pl.fit_transform(X_train)
 X_test = pl.transform(X_test)
 use_feats =(X_train.iloc[0:0].select_dtypes(include=['category','number'])
.columns.tolist() + ['RescuerID','Description'])
 X_train,X_test = X_train[use_feats],X_test[use_feats]
if VERBOSE or SUBMISSION:
logger.info('>> Level 0...')
 X_train,X_test = run_level0(X_train, y_train, X_test)
 if VERBOSE:
logger.info('>> Level 1...')
 X_train,X_test = run_level1(X_train, y_train, X_test)
 if VERBOSE:
logger.info('>> Level 2...')
 X_train,X_test = run_level2(X_train, y_train, X_test)
 if VERBOSE:
logger.info('>> Level 3...')
 y_pred = run_level3(X_train, y_train, X_test)
 if VERBOSE or SUBMISSION:
display(y_pred)
 return y_pred
USE_LB = False
if not SUBMISSION:
validate(X_train, y_train, rescuer_ids)
 if LB:
USE_LB = True
y_pred = run_all(X_train, y_train, X_test)
 msg = 'LB score: %.3f' % qwk(y_pred, y_test)
 logger.info(msg)
 if SUBMISSION and not LB:
y_pred = pd.DataFrame(run_all(X_train, y_train, X_test),
columns=['AdoptionSpeed'],
index=test['PetID'] ).reset_index()
y_pred.to_csv('submission.csv', index=False)
 fig,ax = plt.subplots(1,2,figsize=(16,4))
y_pred['AdoptionSpeed'].hist(ax=ax[0])
 train['AdoptionSpeed'].hist(ax=ax[1])
 ax[0].set_title('Prediction distribution')
 ax[1].set_title('Train set distribution')
 else:
logger.info('Will not make submission without full dataset, turn LB off!' )<set_options>
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print(f'num of train images files: {len(train_image_files)}')
 print(f'num of train metadata files: {len(train_metadata_files)}')
 print(f'num of train sentiment files: {len(train_sentiment_files)}')
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print(f'num of test images files: {len(test_image_files)}')
 print(f'num of test metadata files: {len(test_metadata_files)}')
 print(f'num of test sentiment files: {len(test_sentiment_files)}' )
%matplotlib inline
np.random.seed(seed=1337)
 warnings.filterwarnings('ignore')
 split_char = '/'<define_variables>
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_json_file(self, filename):
with open(filename, 'r', encoding='utf-8')as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns')
 file_sentences_sentiment_df = pd.DataFrame(
 {
'magnitude_sum': file_sentences_sentiment['magnitude'].sum(axis=0),
'score_sum': file_sentences_sentiment['score'].sum(axis=0),
'magnitude_mean': file_sentences_sentiment['magnitude'].mean(axis=0),
'score_mean': file_sentences_sentiment['score'].mean(axis=0),
'magnitude_var': file_sentences_sentiment['magnitude'].var(axis=0),
'score_var': file_sentences_sentiment['score'].var(axis=0),
}, index=[0] 
)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
df_sentiment = pd.concat([df_sentiment, file_sentences_sentiment_df], axis=1)
 df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = f'.. /input/petfinder-adoption-prediction/{mode}_sentiment/{pet_id}.json'
try:
sentiment_file = pet_parser.open_json_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob(f'.. /input/petfinder-adoption-prediction/{mode}_metadata/{pet_id}*.json'))
if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_json_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()
class Config:
num_seed_stratified_group = 4
num_seed_stratified = 4
num_seed_group = 4 
<load_from_csv>
aggregates = ['sum', 'mean', 'var']
sent_agg = ['sum']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 train_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 test_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv' )<import_modules>
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]
import cv2
from PIL import Image as IMG
import os
from keras.applications.densenet import preprocess_input, DenseNet121<define_variables>
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )
img_size = 256
batch_size = 32<feature_engineering>
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False )
pet_ids = train['PetID'].values
im_feature_dict = pd.DataFrame(index=pet_ids, columns=['light_percent', 'dark_percent', 'sharpness',
'top1_size', 'top1_width', 'top1_height'] ).add_prefix('im_')
 for pid in tqdm_notebook(pet_ids):
try:
im, original_shape = load_raw_image(".. /input/petfinder-adoption-prediction/train_images/", pid)
 im_feature_dict.loc[pid, 'im_top1_size'] = original_shape[0] * original_shape[1]
im_feature_dict.loc[pid, 'im_top1_width'] = original_shape[1]
im_feature_dict.loc[pid, 'im_top1_height'] = original_shape[0]
im_feature_dict.loc[pid, 'im_sharpness'] = get_blurrness_score(im)
 im = IMG.fromarray(np.uint8(im))
light_percent, dark_percent = color_analysis(im)
 im_feature_dict.loc[pid, 'im_light_percent'] = light_percent
im_feature_dict.loc[pid, 'im_dark_percent'] = dark_percent
except AttributeError:
pass
pet_ids = test['PetID'].values
test_im_feature_dict = pd.DataFrame(index=pet_ids, columns=['light_percent', 'dark_percent', 'sharpness',
'top1_size', 'top1_width', 'top1_height'] ).add_prefix('im_')
 for pid in tqdm_notebook(pet_ids):
try:
im, original_shape = load_raw_image(".. /input/petfinder-adoption-prediction/test_images/", pid)
 test_im_feature_dict.loc[pid, 'im_top1_size'] = original_shape[0] * original_shape[1]
test_im_feature_dict.loc[pid, 'im_top1_width'] = original_shape[1]
test_im_feature_dict.loc[pid, 'im_top1_height'] = original_shape[0]
test_im_feature_dict.loc[pid, 'im_sharpness'] = get_blurrness_score(im)
 im = IMG.fromarray(np.uint8(im))
light_percent, dark_percent = color_analysis(im)
 test_im_feature_dict.loc[pid, 'im_light_percent'] = light_percent
test_im_feature_dict.loc[pid, 'im_dark_percent'] = dark_percent
except AttributeError:
pass<merge>
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID']
train = train.merge(im_feature_dict, how='left', left_on='PetID', right_index=True)
 print(train.shape)
 train.head()<merge>
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID' )
test = test.merge(test_im_feature_dict, how='left', left_on='PetID', right_index=True)
 print(test.shape)
 test.head()<define_variables>
for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]
img_size = 256
batch_size = 32<define_search_model>
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('none' )
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights=".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )<define_variables>
X_temp['Length_Description'] = X_text['Description'].map(len)
 X_temp['Length_metadata_annots_top_desc'] = X_text['metadata_annots_top_desc'].map(len)
 X_temp['Lengths_sentiment_entities'] = X_text['sentiment_entities'].map(len )
pet_ids = train['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]<create_dataframe>
n_components = 16
text_features = []
for i in X_text.columns:
print(f'generating features from: {i}')
 tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_{}_'.format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )
train_feats = pd.DataFrame.from_dict(features, orient='index')
 train_feats.columns = [f'pic_{i}' for i in range(train_feats.shape[1])]<define_variables>
X_temp = X_temp.merge(img_features, how='left', on='PetID' )
pet_ids = test['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/test_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]<create_dataframe>
train_df_ids = train[['PetID']]
test_df_ids = test[['PetID']]
train_df_imgs = pd.DataFrame(train_image_files)
 train_df_imgs.columns = ['image_filename']
train_imgs_pets = train_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 def getSize(filename):
st = os.stat(filename)
 return st.st_size
def getDimensions(filename):
img_size = Image.open(filename ).size
return img_size
train_df_imgs['image_size'] = train_df_imgs['image_filename'].apply(getSize)
 train_df_imgs['temp_size'] = train_df_imgs['image_filename'].apply(getDimensions)
 train_df_imgs['width'] = train_df_imgs['temp_size'].apply(lambda x : x[0])
 train_df_imgs['height'] = train_df_imgs['temp_size'].apply(lambda x : x[1])
 train_df_imgs = train_df_imgs.drop(['temp_size'], axis=1)
 test_df_imgs['image_size'] = test_df_imgs['image_filename'].apply(getSize)
 test_df_imgs['temp_size'] = test_df_imgs['image_filename'].apply(getDimensions)
 test_df_imgs['width'] = test_df_imgs['temp_size'].apply(lambda x : x[0])
 test_df_imgs['height'] = test_df_imgs['temp_size'].apply(lambda x : x[1])
 test_df_imgs = test_df_imgs.drop(['temp_size'], axis=1)
 aggs = {
'image_size': ['sum', 'mean', 'var'],
'width': ['sum', 'mean', 'var'],
'height': ['sum', 'mean', 'var'],
}
agg_train_imgs = train_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()
agg_test_imgs = test_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index()
agg_imgs = pd.concat([agg_train_imgs, agg_test_imgs], axis=0 ).reset_index(drop=True )
test_feats = pd.DataFrame.from_dict(features, orient='index')
 test_feats.columns = [f'pic_{i}' for i in range(test_feats.shape[1])]<rename_columns>
X_temp = X_temp.merge(agg_imgs, how='left', on='PetID' )
train_feats = train_feats.reset_index()
train_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 test_feats = test_feats.reset_index()
test_feats.rename({'index': 'PetID'}, axis='columns', inplace=True )<concatenate>
X_temp = X_temp.drop(to_drop_columns, axis=1 )
all_ids = pd.concat([train, test], axis=0, ignore_index=True, sort=False)[['PetID']]
all_ids.shape<concatenate>
X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1 )
n_components = 32
svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 features_df = pd.concat([train_feats, test_feats], axis=0)
 features = features_df[[f'pic_{i}' for i in range(256)]].values
svd_col = svd_.fit_transform(features)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('IMG_SVD_')
 img_features = pd.concat([all_ids, svd_col], axis=1 )<load_from_csv>
X_train_non_null.isnull().any().any() , X_test_non_null.isnull().any().any()
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )<define_variables>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print(f'num of train images files: {len(train_image_files)}')
 print(f'num of train metadata files: {len(train_metadata_files)}')
 print(f'num of train sentiment files: {len(train_sentiment_files)}')
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print(f'num of test images files: {len(test_image_files)}')
 print(f'num of test metadata files: {len(test_metadata_files)}')
 print(f'num of test sentiment files: {len(test_sentiment_files)}' )<load_pretrained>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_json_file(self, filename):
with open(filename, 'r', encoding='utf-8')as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns')
 file_sentences_sentiment_df = pd.DataFrame(
 {
'magnitude_sum': file_sentences_sentiment['magnitude'].sum(axis=0),
'magnitude_mean': file_sentences_sentiment['magnitude'].mean(axis=0),
'magnitude_var': file_sentences_sentiment['magnitude'].var(axis=0),
'magnitude_max': file_sentences_sentiment['magnitude'].max(axis=0),
'magnitude_min': file_sentences_sentiment['magnitude'].min(axis=0),
'score_sum': file_sentences_sentiment['score'].sum(axis=0),
'score_mean': file_sentences_sentiment['score'].mean(axis=0),
'score_var': file_sentences_sentiment['score'].var(axis=0),
'score_max': file_sentences_sentiment['score'].max(axis=0),
'score_min': file_sentences_sentiment['score'].min(axis=0),
}, index=[0] 
)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
df_sentiment = pd.concat([df_sentiment, file_sentences_sentiment_df], axis=1)
 df_sentiment['entities'] = file_entities
df_sentiment['language'] = file['language']
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_score_var = np.asarray([x['score'] for x in file_annots] ).var()
file_top_score_max = np.asarray([x['score'] for x in file_annots] ).max()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_score_var = np.nan
file_top_score_max = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_score_var = np.asarray([x['score'] for x in file_colors] ).var()
file_color_score_max = np.asarray([x['score'] for x in file_colors] ).max()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_color_pixelfrac_var = np.asarray([x['pixelFraction'] for x in file_colors] ).var()
file_color_pixelfrac_max = np.asarray([x['pixelFraction'] for x in file_colors] ).max()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'annots_score_var': file_top_score_var,
'annots_score_max': file_top_score_max,
'color_score': file_color_score,
'color_score_var': file_color_score_var,
'color_score_max': file_color_score_max,
'color_pixelfrac': file_color_pixelfrac,
'color_pixelfrac_var': file_color_pixelfrac_var,
'color_pixelfrac_max': file_color_pixelfrac_max,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = f'.. /input/petfinder-adoption-prediction/{mode}_sentiment/{pet_id}.json'
try:
sentiment_file = pet_parser.open_json_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob(f'.. /input/petfinder-adoption-prediction/{mode}_metadata/{pet_id}*.json'))
if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_json_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()<count_values>
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'eta': 0.0123,
'subsample': 0.8,
'colsample_bytree': 0.85,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
}
train_dfs_sentiment.sentiment_language.value_counts()<groupby>
def run_xgb(params, X_train, X_test):
n_splits = 10
verbose_eval = 1000
num_rounds = 60000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test
train_dfs_sentiment.groupby(['PetID'])['sentiment_language'].apply(lambda x: x.mode() ).reset_index().drop(columns='level_1' )<merge>
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null )
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_dfs_lang, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_dfs_lang, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]<merge>
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk )
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )<concatenate>
coefficients_ = coefficients.copy()
coefficients_[0] = 1.66
coefficients_[1] = 2.13
coefficients_[3] = 2.85
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}' )
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False )<define_variables>
Counter(train_predictions )
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName', 'sentiment_language']
to_drop_columns = ['PetID', 'Name', 'RescuerID']<merge>
Counter(test_predictions )
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID' )<feature_engineering>
submission1 = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions} )
for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]<count_values>
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'eta': 0.0123,
'subsample': 0.6,
'colsample_bytree': 0.6,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
}
X['sentiment_language'].value_counts()<count_values>
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null )
X_temp['sentiment_language'].value_counts()<feature_engineering>
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk )
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('none' )<categorify>
coefficients_ = coefficients.copy()
coefficients_[0] = 1.66
coefficients_[1] = 2.13
coefficients_[3] = 2.85
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}' )
X_temp['Length_Description'] = X_text['Description'].map(len)
 X_temp['Length_metadata_annots_top_desc'] = X_text['metadata_annots_top_desc'].map(len)
 X_temp['Lengths_sentiment_entities'] = X_text['sentiment_entities'].map(len )<feature_engineering>
submission1_1 = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions} )
n_components = 16
text_features = []
for i in X_text.columns:
print(f'generating features from: {i}')
 tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_{}_'.format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )<merge>
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'eta': 0.01,
'subsample': 0.8,
'colsample_bytree': 0.6,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
}
X_temp = X_temp.merge(img_features, how='left', on='PetID' )<define_variables>
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null )
train_df_ids = train[['PetID']]
test_df_ids = test[['PetID']]
debut = False
if debug:
train_pet_ids = train_pet_ids[:1000]
test_pet_ids = test_pet_ids[:500]
train_df_imgs = pd.DataFrame(train_image_files)
 train_df_imgs.columns = ['image_filename']
train_imgs_pets = train_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 def getSize(filename):
st = os.stat(filename)
 return st.st_size
def getDimensions(filename):
img_size = Image.open(filename ).size
return img_size
train_df_imgs['image_size'] = train_df_imgs['image_filename'].apply(getSize)
 train_df_imgs['temp_size'] = train_df_imgs['image_filename'].apply(getDimensions)
 train_df_imgs['width'] = train_df_imgs['temp_size'].apply(lambda x : x[0])
 train_df_imgs['height'] = train_df_imgs['temp_size'].apply(lambda x : x[1])
 train_df_imgs = train_df_imgs.drop(['temp_size'], axis=1)
 test_df_imgs['image_size'] = test_df_imgs['image_filename'].apply(getSize)
 test_df_imgs['temp_size'] = test_df_imgs['image_filename'].apply(getDimensions)
 test_df_imgs['width'] = test_df_imgs['temp_size'].apply(lambda x : x[0])
 test_df_imgs['height'] = test_df_imgs['temp_size'].apply(lambda x : x[1])
 test_df_imgs = test_df_imgs.drop(['temp_size'], axis=1)
 aggs = {
'image_size': ['sum', 'mean', 'var', 'max'],
'width': ['sum', 'mean', 'var', 'max'],
'height': ['sum', 'mean', 'var', 'max'],
}
agg_train_imgs = train_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()
agg_test_imgs = test_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index()
agg_imgs = pd.concat([agg_train_imgs, agg_test_imgs], axis=0 ).reset_index(drop=True )<merge>
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk )
X_temp = X_temp.merge(agg_imgs, how='left', on='PetID' )<drop_column>
coefficients_ = coefficients.copy()
coefficients_[0] = 1.66
coefficients_[1] = 2.13
coefficients_[3] = 2.85
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}' )
X_temp = X_temp.drop(to_drop_columns, axis=1 )<define_variables>
submission1_2 = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions} )
X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1 )<count_missing_values>
%matplotlib inline
np.random.seed(seed=1337)
 warnings.filterwarnings('ignore')
 split_char = '/'
os.listdir('.. /input' )
X_train_non_null.isnull().any().any() , X_test_non_null.isnull().any().any()<categorify>
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv' )
def target_encoding(count_offset, X_te_train, X_te_valid, X_te_test, sigm_scale_divider=5):
categorical_column_list = ["Breed1", "Breed2", "Gender", "Color1", "Color2", "Color3", "Health", "State",
'sentiment_language']
target_encoding_columns = []
mean_all = X_te_train.AdoptionSpeed.mean()
for categ_column in categorical_column_list:
df_categorical = X_te_train[[categ_column, "AdoptionSpeed"]]
df_categorical = df_categorical.fillna(-1 ).astype(int)
 te_df = df_categorical.groupby(categ_column ).mean()
df_count = df_categorical.groupby(categ_column ).count()
sigmoid_scale =(count_offset - 1)/ sigm_scale_divider
smoothing_coef = 1 /(1 + np.exp(-(df_count - count_offset)/ sigmoid_scale))
target_enc_df = mean_all *(1 - smoothing_coef)+ te_df * smoothing_coef
target_enc_df = target_enc_df.add_prefix(f'TargetEncoding_{categ_column}_')
 X_te_train = pd.merge(X_te_train, target_enc_df.reset_index() , on=categ_column, how='left')
 X_te_valid = pd.merge(X_te_valid, target_enc_df.reset_index() , on=categ_column, how='left')
 X_te_test = pd.merge(X_te_test, target_enc_df.reset_index() , on=categ_column, how='left')
 return X_te_train, X_te_valid, X_te_test<import_modules>
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv')
 train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print(f'num of train images files: {len(train_image_files)}')
 print(f'num of train metadata files: {len(train_metadata_files)}')
 print(f'num of train sentiment files: {len(train_sentiment_files)}')
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print(f'num of test images files: {len(test_image_files)}')
 print(f'num of test metadata files: {len(test_metadata_files)}')
 print(f'num of test sentiment files: {len(test_sentiment_files)}')
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<compute_train_metric>
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_json_file(self, filename):
with open(filename, 'r', encoding='utf-8')as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns')
 file_sentences_sentiment_df = pd.DataFrame(
 {
'magnitude_sum': file_sentences_sentiment['magnitude'].sum(axis=0),
'score_sum': file_sentences_sentiment['score'].sum(axis=0),
}, index=[0] 
)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
df_sentiment = pd.concat([df_sentiment, file_sentences_sentiment_df], axis=1)
 df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = f'.. /input/petfinder-adoption-prediction/{mode}_sentiment/{pet_id}.json'
try:
sentiment_file = pet_parser.open_json_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob(f'.. /input/petfinder-adoption-prediction/{mode}_metadata/{pet_id}*.json'))
if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_json_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()
def rounder(y, thresholds=[0.5, 1.5, 2.5, 3.5]):
return np.digitize(y, thresholds)
 class OptimizedRounder(object):
def __init__(self):
self.coef_ = [0.5, 1.5, 2.5, 3.5]
def _loss(self, coef, X, y):
X_p = rounder(X, np.sort(coef))
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
coef = self.coef_
for it1 in range(10):
for idx in range(4):
golden1 = 2/(3+np.sqrt(5))
golden2 = 1 - golden1
a = 0.0 if idx-1 < 0 else coef[idx-1]
b = 4.0 if idx+1 > 3 else coef[idx+1]
coef[idx] = a +(b-a)*golden1
la = self._loss(coef, X, y)
 coef[idx] = a +(b-a)*golden2
lb = self._loss(coef, X, y)
 for it2 in range(20):
if la > lb:
a = a +(b-a)*golden1
coef[idx] = a +(b-a)*golden2
la = lb
lb = self._loss(coef, X, y)
 else:
b = a +(b-a)*golden2
coef[idx] = a +(b-a)*golden1
lb = la
la = self._loss(coef, X, y)
 self.coef_ = coef
def predict(self, X, coef):
return rounder(X, coef)
 def coefficients(self):
return self.coef_<groupby>
aggregates = ['sum', 'mean']
sent_agg = ['sum']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 train_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 test_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()
def stratified_group_k_fold(X, y, groups, k, seed=None):
labels_num = np.max(y)+ 1
y_counts_per_group = defaultdict(lambda: np.zeros(labels_num))
y_distr = Counter()
for label, g in zip(y, groups):
y_counts_per_group[g][label] += 1
y_distr[label] += 1
y_counts_per_fold = defaultdict(lambda: np.zeros(labels_num))
groups_per_fold = defaultdict(set)
 def eval_y_counts_per_fold(y_counts, fold):
y_counts_per_fold[fold] += y_counts
std_per_label = []
for label in range(labels_num):
label_std = np.std([y_counts_per_fold[i][label] / y_distr[label] for i in range(k)])
 std_per_label.append(label_std)
 y_counts_per_fold[fold] -= y_counts
return np.mean(std_per_label)
 groups_and_y_counts = list(y_counts_per_group.items())
 random.Random(seed ).shuffle(groups_and_y_counts)
 for g, y_counts in sorted(groups_and_y_counts, key=lambda x: -np.std(x[1])) :
best_fold = None
min_eval = None
for i in range(k):
fold_eval = eval_y_counts_per_fold(y_counts, i)
 if min_eval is None or fold_eval < min_eval:
min_eval = fold_eval
best_fold = i
y_counts_per_fold[best_fold] += y_counts
groups_per_fold[best_fold].add(g)
 all_groups = set(groups)
 for i in range(k):
train_groups = all_groups - groups_per_fold[i]
test_groups = groups_per_fold[i]
train_indices = [i for i, g in enumerate(groups)if g in train_groups]
test_indices = [i for i, g in enumerate(groups)if g in test_groups]
yield train_indices, test_indices<train_model>
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]
train_gid = train.RescuerID.values
def run_xgb(params, X_train, X_test, cv):
n_splits = 10
verbose_eval = 10000
num_rounds = 5000
early_stop = 500
try:
kf = cv(n_splits=n_splits, shuffle=True)
 except:
kf = cv(n_splits=n_splits)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
importance_list = []
X_test_original = X_test.copy()
for train_idx, valid_idx in tqdm(kf.split(X_train, X_train['AdoptionSpeed'].values, train_gid)) :
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
X_tr, X_val, X_test = target_encoding(params['count_offset'], X_tr, X_val, X_test_original, params['sigm_scale_divider'])
 y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
importance_list.append(model.get_score(importance_type='gain'))
i += 1
return model, oof_train, oof_test.mean(axis=1)
 def run_xgb_stratified_group(params, X_train, X_test):
n_splits = 10
verbose_eval = 10000
num_rounds = 5000
early_stop = 500
oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
importance_list = []
X_test_original = X_test.copy()
for train_idx, valid_idx in tqdm(stratified_group_k_fold(X_train, X_train['AdoptionSpeed'].values.astype(np.int8), train_gid, k=n_splits)) :
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
X_tr, X_val, X_test = target_encoding(params['count_offset'], X_tr, X_val, X_test_original, params['sigm_scale_divider'])
 y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
importance_list.append(model.get_score(importance_type='gain'))
i += 1
return model, oof_train, oof_test.mean(axis=1 )<init_hyperparams>
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape)
oof_train_list = []
oof_test_list = []
for i_seed in range(Config.num_seed_stratified):
print(i_seed)
 gc.collect()
xgb_params = {
'eval_metric': 'rmse',
'seed': i_seed,
'eta': 0.02042285921574787,
'gamma': 0.0006332233837072564,
'max_depth': 4,
'min_child_weight': 12,
'subsample': 0.8928595200024736,
'colsample_bytree': 0.3894222269910513,
'colsample_bylevel': 0.7,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
'count_offset': 40,
'sigm_scale_divider': 50
}
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null, StratifiedKFold)
 oof_train_list.append(oof_train)
 oof_test_list.append(oof_test)
 for i_seed in range(Config.num_seed_group):
print(i_seed)
 gc.collect()
xgb_params = {
'eval_metric': 'rmse',
'seed': i_seed,
'eta': 0.02042285921574787,
'gamma': 0.0006332233837072564,
'max_depth': 4,
'min_child_weight': 12,
'subsample': 0.8928595200024736,
'colsample_bytree': 0.3894222269910513,
'colsample_bylevel': 0.7,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
'count_offset': 40,
'sigm_scale_divider': 50
}
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null, GroupKFold)
 oof_train_list.append(oof_train)
 oof_test_list.append(oof_test)
 for i_seed in range(Config.num_seed_stratified_group):
print(i_seed)
 gc.collect()
xgb_params = {
'eval_metric': 'rmse',
'seed': i_seed,
'eta': 0.02042285921574787,
'gamma': 0.0006332233837072564,
'max_depth': 4,
'min_child_weight': 12,
'subsample': 0.8928595200024736,
'colsample_bytree': 0.3894222269910513,
'colsample_bylevel': 0.7,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
'count_offset': 40,
'sigm_scale_divider': 50
}
model, oof_train, oof_test = run_xgb_stratified_group(xgb_params, X_train_non_null, X_test_non_null)
 oof_train_list.append(oof_train)
 oof_test_list.append(oof_test)
 oof_train = np.mean(oof_train_list, axis=0)
 oof_test = np.mean(oof_test_list, axis=0)
<prepare_x_and_y>
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False)
 X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID']
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID')
 for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('none' )
xgb_oof_train, xgb_oof_test = oof_train, oof_test<find_best_model_class>
n_components = 5
text_features = []
for i in X_text.columns:
print(f'generating features from: {i}')
 tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_{}_'.format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk)
 coefficients_ = optR.coefficients()
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(oof_test, coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}' )<import_modules>
X_temp = X_temp.drop(to_drop_columns, axis=1)
 X_train = X_temp.loc[np.isfinite(X_temp.AdoptionSpeed), :]
X_test = X_temp.loc[~np.isfinite(X_temp.AdoptionSpeed), :]
X_test = X_test.drop(['AdoptionSpeed'], axis=1)
 assert X_train.shape[0] == train.shape[0]
assert X_test.shape[0] == test.shape[0]
train_cols = X_train.columns.tolist()
train_cols.remove('AdoptionSpeed')
 test_cols = X_test.columns.tolist()
assert np.all(train_cols == test_cols)
 X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1 )
import lightgbm as lgb<init_hyperparams>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )
oof_train_list = []
oof_test_list = []
for i_seed in tqdm(range(3)) :
gc.collect()
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'rmse',
'eta': 0.01235408383482946,
'subsample': 0.90892648949676,
'colsample_bytree': 0.3595323228044939,
'min_split_gain': 0.0017213784304419208,
'min_child_weight': 2.489294481302874e-05,
'min_child_samples': 3,
'n_estimators': 1838,
'max_depth': 12,
'count_offset': 21.197455354403893,
'sigm_scale_divider': 49.03532266115622,
'bagging_freq': 4,
'verbose': -1,
'silent': 1,
'n_jobs': -1,
'seed': i_seed
}
model, oof_train, oof_test = run_lgbm(lgb_params, X_train_non_null, X_test_non_null, StratifiedKFold)
 oof_train_list.append(oof_train)
 oof_test_list.append(oof_test)
 for i_seed in tqdm(range(3)) :
gc.collect()
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'rmse',
'eta': 0.01235408383482946,
'subsample': 0.90892648949676,
'colsample_bytree': 0.3595323228044939,
'min_split_gain': 0.0017213784304419208,
'min_child_weight': 2.489294481302874e-05,
'min_child_samples': 3,
'n_estimators': 1838,
'max_depth': 12,
'count_offset': 21.197455354403893,
'sigm_scale_divider': 49.03532266115622,
'bagging_freq': 4,
'verbose': -1,
'silent': 1,
'n_jobs': -1,
'seed': i_seed
}
model, oof_train, oof_test = run_lgbm(lgb_params, X_train_non_null, X_test_non_null, GroupKFold)
 oof_train_list.append(oof_train)
 oof_test_list.append(oof_test)
 for i_seed in tqdm(range(3)) :
gc.collect()
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'rmse',
'eta': 0.01235408383482946,
'subsample': 0.90892648949676,
'colsample_bytree': 0.3595323228044939,
'min_split_gain': 0.0017213784304419208,
'min_child_weight': 2.489294481302874e-05,
'min_child_samples': 3,
'n_estimators': 1838,
'max_depth': 12,
'count_offset': 21.197455354403893,
'sigm_scale_divider': 49.03532266115622,
'bagging_freq': 4,
'verbose': -1,
'silent': 1,
'n_jobs': -1,
'seed': i_seed
}
model, oof_train, oof_test = run_lgbm_stratified_group(lgb_params, X_train_non_null, X_test_non_null)
 oof_train_list.append(oof_train)
 oof_test_list.append(oof_test)
 oof_train = np.mean(oof_train_list, axis=0)
 oof_test = np.mean(oof_test_list, axis=0 )<find_best_model_class>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk)
 coefficients_ = optR.coefficients()
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(oof_test, coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}' )<define_variables>
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'silent': 1,
}
def run_xgb(params, X_train, X_test):
n_splits = 5
verbose_eval = 1000
num_rounds = 30000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null )
lgb_oof_train = oof_train
lgb_oof_test = oof_test<statistical_test>
xgb_params = {
'eval_metric': 'rmse',
'seed': 2337,
'silent': 1,
}
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk )
best_qwk = 0
for weight in np.arange(0, 1.01, 0.1):
print(weight)
 oof_train = weight * xgb_oof_train +(1 - weight)* lgb_oof_train
oof_test = weight * xgb_oof_test +(1 - weight)* lgb_oof_test
plot_pred(oof_train)
 plot_pred(oof_test)
 optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk)
 coefficients_ = optR.coefficients()
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(oof_test, coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}')
 if qwk > best_qwk:
best_weight = weight
best_qwk = qwk
best_test_predictions = test_predictions
print(best_weight, best_qwk )<save_to_csv>
coefficients_ = coefficients.copy()
coefficients_[0] = 1.65
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_ ).astype(np.int8)
 submission2_1 = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions} )
submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': best_test_predictions})
 submission.to_csv('submission.csv', index=False)
 submission.head()<set_options>
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null)
 optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk)
 coefficients_ = coefficients.copy()
coefficients_[0] = 1.65
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_ ).astype(np.int8)
 submission2 = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions} )
%matplotlib inline
np.random.seed(seed=1337)
 warnings.filterwarnings('ignore')
 split_char = '/'<load_from_csv>
train_df = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 img_size = 256
batch_size = 16
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv' )<import_modules>
pet_ids = train_df['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
import cv2
import os
from keras.applications.densenet import preprocess_input, DenseNet121<define_variables>
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights = ".. /input/densenet121weights/densenet121_weights_tf_dim_ordering_tf_kernels_notop.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )
img_size = 256
batch_size = 256<define_search_model>
features = {}
for b in tqdm_notebook(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights=".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )<define_variables>
train_feats = pd.DataFrame.from_dict(features, orient='index' )
pet_ids = train['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]<create_dataframe>
test_df = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 pet_ids = test_df['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm_notebook(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/test_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
test_feats = pd.DataFrame.from_dict(features, orient='index' )
train_feats = pd.DataFrame.from_dict(features, orient='index')
 train_feats.columns = [f'pic_{i}' for i in range(train_feats.shape[1])]<define_variables>
train_feats.reset_index(inplace=True)
 test_feats.reset_index(inplace=True )
pet_ids = test['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/test_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]<create_dataframe>
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv')
 labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv')
 train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
train_img_features = train_feats
test_img_features = test_feats
col_names =["PetID"] + ["{}_img_feature".format(_)for _ in range(256)]
train_img_features.columns = col_names
test_img_features.columns = col_names
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_metadata_file(self, filename):

 with open(filename, 'r')as f:
metadata_file = json.load(f)
 return metadata_file
def open_sentiment_file(self, filename):

 with open(filename, 'r')as f:
sentiment_file = json.load(f)
 return sentiment_file
def open_image_file(self, filename):

 image = np.asarray(Image.open(filename))
return image
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 if self.extract_sentiment_text:
file_sentences_text = [x['text']['content'] for x in file['sentences']]
file_sentences_text = self.sentence_sep.join(file_sentences_text)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns' ).sum()
file_sentences_sentiment = file_sentences_sentiment.add_prefix('document_' ).to_dict()
file_sentiment.update(file_sentences_sentiment)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
if self.extract_sentiment_text:
df_sentiment['text'] = file_sentences_text
df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations'][:int(len(file['labelAnnotations'])* 0.3)]
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = '.. /input/petfinder-adoption-prediction/{}_sentiment/{}.json'.format(mode, pet_id)
 try:
sentiment_file = pet_parser.open_sentiment_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob('.. /input/petfinder-adoption-prediction/{}_metadata/{}*.json'.format(mode, pet_id)))
 if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_metadata_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
def agg_features(df_metadata, df_sentiment):
aggregates = ['mean', "median", 'sum', "var", "std", "min", "max", "nunique"]
metadata_desc = df_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
metadata_desc = metadata_desc.reset_index()
metadata_desc['metadata_annots_top_desc'] = metadata_desc['metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
metadata_gr = df_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in metadata_gr.columns:
if 'PetID' not in i:
metadata_gr[i] = metadata_gr[i].astype(float)
 metadata_gr = metadata_gr.groupby(['PetID'] ).agg(aggregates)
 metadata_gr.columns = pd.Index(['{}_{}_{}'.format(prefix, c[0], c[1].upper())for c in metadata_gr.columns.tolist() ])
 metadata_gr = metadata_gr.reset_index()
sentiment_desc = df_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
sentiment_desc = sentiment_desc.reset_index()
sentiment_desc['sentiment_entities'] = sentiment_desc['sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
sentiment_gr = df_sentiment.drop(['sentiment_entities'], axis=1)
 for i in sentiment_gr.columns:
if 'PetID' not in i:
sentiment_gr[i] = sentiment_gr[i].astype(float)
 sentiment_gr = sentiment_gr.groupby(['PetID'] ).agg(aggregates)
 sentiment_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in sentiment_gr.columns.tolist() ])
 sentiment_gr = sentiment_gr.reset_index()
return sentiment_gr, metadata_gr, metadata_desc, sentiment_desc
def breed_features(df, _labels_breed):
breed_main = df[['Breed1']].merge(_labels_breed, how='left', left_on='Breed1', right_on='BreedID', suffixes=('', '_main_breed'))
breed_main = breed_main.iloc[:, 2:]
breed_main = breed_main.add_prefix('main_breed_')
 breed_second = df[['Breed2']].merge(_labels_breed, how='left', left_on='Breed2', right_on='BreedID', suffixes=('', '_second_breed'))
breed_second = breed_second.iloc[:, 2:]
breed_second = breed_second.add_prefix('second_breed_')
 return breed_main, breed_second
def impact_coding(data, feature, target='y'):

 n_folds = 20
n_inner_folds = 10
impact_coded = pd.Series()
oof_default_mean = data[target].mean()
kf = KFold(n_splits=n_folds, shuffle=True)
 oof_mean_cv = pd.DataFrame()
split = 0
for infold, oof in kf.split(data[feature]):
impact_coded_cv = pd.Series()
kf_inner = KFold(n_splits=n_inner_folds, shuffle=True)
 inner_split = 0
inner_oof_mean_cv = pd.DataFrame()
oof_default_inner_mean = data.iloc[infold][target].mean()
for infold_inner, oof_inner in kf_inner.split(data.iloc[infold]):
oof_mean = data.iloc[infold_inner].groupby(by=feature)[target].mean()
impact_coded_cv = impact_coded_cv.append(data.iloc[infold].apply(
 lambda x: oof_mean[x[feature]]
if x[feature] in oof_mean.index
else oof_default_inner_mean
, axis=1))
inner_oof_mean_cv = inner_oof_mean_cv.join(pd.DataFrame(oof_mean), rsuffix=inner_split, how='outer')
 inner_oof_mean_cv.fillna(value=oof_default_inner_mean, inplace=True)
 inner_split += 1
oof_mean_cv = oof_mean_cv.join(pd.DataFrame(inner_oof_mean_cv), rsuffix=split, how='outer')
 oof_mean_cv.fillna(value=oof_default_mean, inplace=True)
 split += 1
impact_coded = impact_coded.append(data.iloc[oof].apply(
 lambda x: inner_oof_mean_cv.loc[x[feature]].mean()
if x[feature] in inner_oof_mean_cv.index
else oof_default_mean
, axis=1))
return impact_coded, oof_mean_cv.mean(axis=1), oof_default_mean
def frequency_encoding(df, col_name):
new_name = "{}_counts".format(col_name)
 new_col_name = "{}_freq".format(col_name)
 grouped = df.groupby(col_name ).size().reset_index(name=new_name)
 df = df.merge(grouped, how = "left", on = col_name)
 df[new_col_name] = df[new_name]/df[new_name].count()
del df[new_name]
return df
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator)
 class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))
def train_lightgbm(X_train, X_test, params, n_splits, num_rounds, verbose_eval, early_stop):
kfold = StratifiedKFold(n_splits=n_splits, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_index, valid_index in kfold.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_index, :]
X_val = X_train.iloc[valid_index, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 print('
y_tr distribution: {}'.format(Counter(y_tr)))
 d_train = lgb.Dataset(X_tr, label=y_tr)
 d_valid = lgb.Dataset(X_val, label=y_val)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 val_pred = model.predict(X_val, num_iteration=model.best_iteration)
 test_pred = model.predict(X_test, num_iteration=model.best_iteration)
 oof_train[valid_index] = val_pred
oof_test[:, i] = test_pred
i += 1
return oof_train, oof_test
pet_parser = PetFinderParser()

 train_sentiment_gr, train_metadata_gr, train_metadata_desc, train_sentiment_desc = agg_features(train_dfs_metadata, train_dfs_sentiment)
 test_sentiment_gr, test_metadata_gr, test_metadata_desc, test_sentiment_desc = agg_features(test_dfs_metadata, test_dfs_sentiment)
 train_proc = train.copy()
for tr in [train_sentiment_gr, train_metadata_gr, train_metadata_desc, train_sentiment_desc]:
train_proc = train_proc.merge(tr, how='left', on='PetID')
 test_proc = test.copy()
for ts in [test_sentiment_gr, test_metadata_gr, test_metadata_desc, test_sentiment_desc]:
test_proc = test_proc.merge(
 ts, how='left', on='PetID')
 train_proc = pd.merge(train_proc, train_img_features, on="PetID")
 test_proc = pd.merge(test_proc, test_img_features, on="PetID")
 train_breed_main, train_breed_second = breed_features(train_proc, labels_breed)
 train_proc = pd.concat([train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main, test_breed_second = breed_features(test_proc, labels_breed)
 test_proc = pd.concat([test_proc, test_breed_main, test_breed_second], axis=1)
 X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False)
 column_types = X.dtypes
int_cols = column_types[column_types == 'int']
float_cols = column_types[column_types == 'float']
cat_cols = column_types[column_types == 'object']
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID']
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID')
 for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('<MISSING>')
 n_components = 5
text_features = []
for i in X_text.columns:
print('generating features from: {}'.format(i))
svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 nmf_ = NMF(
 n_components=n_components, random_state=1337)
 tfidf_col = TfidfVectorizer().fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('SVD_{}_'.format(i))
nmf_col = nmf_.fit_transform(tfidf_col)
 nmf_col = pd.DataFrame(nmf_col)
 nmf_col = nmf_col.add_prefix('NMF_{}_'.format(i))
text_features.append(svd_col)
 text_features.append(nmf_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1)
 X_temp["name_length"] = X_temp.Name[X_temp.Name.isnull() ].map(lambda x: len(str(x)))
 X_temp["name_length"] = X_temp.Name.map(lambda x: len(str(x)))
 X_temp = X_temp.drop(to_drop_columns, axis=1)
 X_train = X_temp.loc[np.isfinite(X_temp.AdoptionSpeed), :]
X_test = X_temp.loc[~np.isfinite(X_temp.AdoptionSpeed), :]
X_test = X_test.drop(['AdoptionSpeed'], axis=1)
 print('X_train shape: {}'.format(X_train.shape))
print('X_test shape: {}'.format(X_test.shape))
assert X_train.shape[0] == train.shape[0]
assert X_test.shape[0] == test.shape[0]
train_cols = X_train.columns.tolist()
train_cols.remove('AdoptionSpeed')
 test_cols = X_test.columns.tolist()
np.random.seed(13)
 categorical_features = ["Type", "Breed1", "Breed2", "Color1" ,"Color2", "Color3", "State"]
impact_coding_map = {}
for f in categorical_features:
print("Impact coding for {}".format(f))
X_train["impact_encoded_{}".format(f)], impact_coding_mapping, default_coding = impact_coding(X_train, f, target="AdoptionSpeed")
 impact_coding_map[f] =(impact_coding_mapping, default_coding)
 mapping, default_mean = impact_coding_map[f]
X_test["impact_encoded_{}".format(f)] = X_test.apply(lambda x: mapping[x[f]] if x[f] in mapping
else default_mean, axis=1)
 for cat in categorical_features:
X_train = frequency_encoding(X_train, cat)
 X_test = frequency_encoding(X_test, cat)
 params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 70,
'max_depth': 9,
'learning_rate': 0.01,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.02,
'min_child_samples': 150,
'min_child_weight': 0.02,
'lambda_l2': 0.0475,
'verbosity': -1,
'data_random_seed': 17}
early_stop = 500
verbose_eval = 1000
num_rounds = 10000
n_splits = 5
oof_train, oof_test = train_lightgbm(X_train, X_test, params, n_splits, num_rounds, verbose_eval, early_stop)
 optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(oof_train, coefficients)
 print("
Valid Counts = ", Counter(X_train['AdoptionSpeed'].values))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, pred_test_y_k)
 print("QWK = ", qwk)
 coefficients_ = coefficients.copy()
coefficients_[0] = 1.645
coefficients_[1] = 2.115
coefficients_[3] = 2.84
train_predictions = optR.predict(oof_train, coefficients_ ).astype(int)
 print('train pred distribution: {}'.format(Counter(train_predictions)))
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_)
 print('test pred distribution: {}'.format(Counter(test_predictions)))
 submission3 = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions.astype(np.int32)})
test_feats = pd.DataFrame.from_dict(features, orient='index')
 test_feats.columns = [f'pic_{i}' for i in range(test_feats.shape[1])]<rename_columns>
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 70,
'max_depth': 9,
'learning_rate': 0.005,
'bagging_fraction': 0.75,
'feature_fraction': 0.7,
'min_split_gain': 0.04,
'min_child_samples': 150,
'min_child_weight': 0.02,
'lambda_l2': 0.0475,
'verbosity': -1,
'data_random_seed': 17}
oof_train, oof_test = train_lightgbm(X_train, X_test, params, n_splits, num_rounds, verbose_eval, early_stop)
 optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(oof_train, coefficients)
 print("
Valid Counts = ", Counter(X_train['AdoptionSpeed'].values))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, pred_test_y_k)
 print("QWK = ", qwk)
 coefficients_ = coefficients.copy()
coefficients_[0] = 1.645
coefficients_[1] = 2.115
coefficients_[3] = 2.84
train_predictions = optR.predict(oof_train, coefficients_ ).astype(int)
 print('train pred distribution: {}'.format(Counter(train_predictions)))
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_)
 print('test pred distribution: {}'.format(Counter(test_predictions)))
 submission3_1 = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions.astype(np.int32)} )
train_feats = train_feats.reset_index()
train_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 test_feats = test_feats.reset_index()
test_feats.rename({'index': 'PetID'}, axis='columns', inplace=True )<concatenate>
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 100,
'max_depth': 5,
'learning_rate': 0.003,
'bagging_fraction': 0.65,
'feature_fraction': 0.6,
'min_split_gain': 0.04,
'min_child_samples': 150,
'min_child_weight': 0.01,
'lambda_l2': 0.03,
'verbosity': -1,
'data_random_seed': 17}
oof_train, oof_test = train_lightgbm(X_train, X_test, params, n_splits, num_rounds, verbose_eval, early_stop)
 optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(oof_train, coefficients)
 print("
Valid Counts = ", Counter(X_train['AdoptionSpeed'].values))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, pred_test_y_k)
 print("QWK = ", qwk)
 coefficients_ = coefficients.copy()
coefficients_[0] = 1.645
coefficients_[1] = 2.115
coefficients_[3] = 2.84
train_predictions = optR.predict(oof_train, coefficients_ ).astype(int)
 print('train pred distribution: {}'.format(Counter(train_predictions)))
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_)
 print('test pred distribution: {}'.format(Counter(test_predictions)))
 submission3_2 = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions.astype(np.int32)} )
<concatenate><EOS> 
submission1["AdoptionSpeed"] = submission1.AdoptionSpeed*0.125 + submission1_1.AdoptionSpeed*0.125 + submission1_2.AdoptionSpeed*0.125 + submission2.AdoptionSpeed*0.125 + submission2_1.AdoptionSpeed*0.125 + submission3.AdoptionSpeed*0.125 + submission3_1.AdoptionSpeed*0.125 + submission3_1.AdoptionSpeed*0.125
submission1["AdoptionSpeed"] = submission1["AdoptionSpeed"].round().astype(int)
 submission1.to_csv('submission.csv', index=False )