Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<drop_column>
SUBMISSION = True
USE_RESCUER_CV = not SUBMISSION
LB = not SUBMISSION
SEED = 209321206
VERBOSE = 1
%%time
def run_xgb(params, X_train, X_test):
n_splits = 10
verbose_eval = 1000
num_rounds = 60000
early_stop = 500
if 'RescuerID' in X_train.columns:
X_train = X_train.drop(to_drop_columns, axis=1)
 X_test = X_test.drop(to_drop_columns, axis=1)
 if 'Pet_Breed' in X_train.columns:
X_train = X_train.drop([ 'RescuerID_COUNT'], axis=1)
 X_test = X_test.drop([ 'RescuerID_COUNT'], axis=1)
 kf = KFold(n_splits=n_splits, shuffle=True, random_state=34535)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null)
 optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk)
 coefficients_ = coefficients.copy()
coefficients_[0] = 1.66
coefficients_[1] = 2.13
coefficients_[3] = 2.85
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}')
 print(Counter(train_predictions), Counter(test_predictions))
submission5 = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions})
 submission5.to_csv('submission5.csv', index=False )<save_to_csv>
START_TIME = pd.datetime.now()
submission1["AdoptionSpeed"] = submission1.AdoptionSpeed*0.4 + submission4.AdoptionSpeed*0.1 + submission2.AdoptionSpeed*0.20 + submission3.AdoptionSpeed*0.20+submission5.AdoptionSpeed*0.1
submission1["AdoptionSpeed"] = submission1["AdoptionSpeed"].round().astype(int)
 submission1.to_csv('submission.csv', index=False )<import_modules>
class KaggleHandler(StreamHandler):
def __init__(self):
StreamHandler.__init__(self)
 def emit(self, record):
msg_fmt = re.sub(r'[^0-9A-z\.\,\-\:\=]', ' ', str(self.format(record)))
 secs_passed =(pd.datetime.now() - START_TIME ).seconds
msg = '[%ss] %s' %(secs_passed, msg_fmt)
 os.system(f'echo {msg}')
 sys.stdout.write(msg + '
')
 logger = logging.getLogger(__name__)
 logger.handlers = []
logger.setLevel(logging.DEBUG)
 fmt = logging.Formatter('[%(asctime)s]: %(message)s')
 kh = KaggleHandler()
kh.setFormatter(fmt)
 logger.addHandler(kh )
%matplotlib inline
np.random.seed(seed=1337)
 warnings.filterwarnings('ignore')
 tqdm.pandas(tqdm_notebook)
 split_char = '/'<load_from_csv>
logger.info('Seed set to %d' % SEED)
 np.random.seed(SEED )
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv' )<split>
def qwk(predicted_ratings, actual_ratings, nb_ratings=5):

 assert len(actual_ratings)== len(predicted_ratings)
 actual_ratings = actual_ratings.astype(np.int64)
 predicted_ratings = predicted_ratings.astype(np.int64)
 conf_mtx = confusion_matrix(actual_ratings, predicted_ratings ).astype(np.float64)
 weight_mtx = np.zeros(( nb_ratings, nb_ratings))
for i in range(nb_ratings):
for j in range(nb_ratings):
weight_mtx[i][j] =(i-j)**2 /(( nb_ratings-1)**2)
 actual_ratings_hist = np.bincount(actual_ratings,minlength=nb_ratings)
 predicted_ratings_hist = np.bincount(predicted_ratings, minlength=nb_ratings)
 out_prod = np.outer(actual_ratings_hist, predicted_ratings_hist ).astype(np.float64)
 conf_mtx /= conf_mtx.sum()
out_prod /= out_prod.sum()
numerator =(conf_mtx * weight_mtx ).sum()
denominator =(out_prod * weight_mtx ).sum()
score =(1 -(numerator/denominator))
return score
debug = False
if debug:
train = train[:500]
test = test[:200]<import_modules>
def validate(X, y, rescuer_ids, n_cv=8, n_folds=5):

 df_preds = []
cv_scores = []
cv_accs = []
global CV,FOLD
for j in range(n_cv):
CV = j
skf = StratifiedKFold(n_folds, shuffle=True, random_state=SEED+j)
 if USE_RESCUER_CV:
rescuer_idx = rescuer_ids['unique'].values
cv_gen = skf.split(rescuer_idx, rescuer_ids['count'].values)
 else:
cv_gen = skf.split(X, y)
 scores = []
accs = []
for i,(train_idx,test_idx)in enumerate(cv_gen):
FOLD = i
if USE_RESCUER_CV:
train_idx = np.array([idx for l in rescuer_idx[train_idx] for idx in l])
 test_idx = np.array([idx for l in rescuer_idx[test_idx] for idx in l])
 X_train,y_train = X.iloc[train_idx],y.iloc[train_idx]
X_test,y_test = X.iloc[test_idx],y.iloc[test_idx]
y_pred = run_all(X_train, y_train, X_test)
 score = qwk(y_pred, y_test)
 scores.append(score)
 acc = [-1, -1, -1, -1, -1]
if VERBOSE:
print(classification_report(y_test, y_pred))
acc = plot_confusion_matrix(y_test, y_pred)
 accs.append(acc)
 logger.info('CV %d score: %.3f(running mean %.3f and std %.3f)- Running acc.%s' % 
(i+1, score, np.mean(scores), np.std(scores), np.around(np.array(accs ).mean(axis=0), 1)))
 df_pred = pd.DataFrame()
df_pred['Prediction'] = y_pred
df_pred['Run'] = j
df_pred['Fold'] = i
df_pred['PetID'] = train['PetID'].values[test_idx]
df_preds.append(df_pred)
 cv_scores.append(scores)
 cv_accs.append(accs)
 cv_means = np.mean(cv_scores, axis=1)
 logger.info('> Running CV mean after %d runs: %.3f [%.3f - %.3f](std %.3f)- Individual [%.3f - %.3f] - Accs: %s' % 
(j+1,
np.mean(cv_means), np.min(cv_means), np.max(cv_means), np.std(cv_means),
np.min(cv_scores), np.max(cv_scores), np.around(np.array(cv_accs ).mean(axis=1 ).mean(axis=0), 1)))
 pd.concat(df_preds, axis=0 ).to_csv('df_preds.csv', index=False)
 all_cv_mean = np.mean(cv_means)
 return all_cv_mean
import cv2
import os
from keras.applications.densenet import preprocess_input, DenseNet121<define_variables>
class PandasTransform(TransformerMixin, BaseEstimator):
def __init__(self, fn):
self.fn = fn
def fit(self, X, y=None):
return self
def transform(self, X, y=None, copy=None):
return self.fn(X)
 class PandasFeatureUnion(FeatureUnion):
def fit_transform(self, X, y=None, **fit_params):
self._validate_transformers()
result = Parallel(n_jobs=self.n_jobs )(
 delayed(_fit_transform_one )(
 transformer=trans,
X=X.copy() ,
y=y,
weight=weight,
**fit_params)
 for name, trans, weight in self._iter())
 if not result:
return np.zeros(( X.shape[0], 0))
Xs, transformers = zip(*result)
 self._update_transformer_list(transformers)
 if any(sparse.issparse(f)for f in Xs):
Xs = sparse.hstack(Xs ).tocsr()
else:
Xs = self.merge_dataframes_by_column(Xs)
 return Xs
def merge_dataframes_by_column(self, Xs):
return pd.concat(Xs, axis="columns", copy=False)
 def transform(self, X):
Xs = Parallel(n_jobs=self.n_jobs )(
 delayed(_transform_one )(
 transformer=trans,
X=X.copy() ,
y=None,
weight=weight)
 for name, trans, weight in self._iter())
 if not Xs:
return np.zeros(( X.shape[0], 0))
if any(sparse.issparse(f)for f in Xs):
Xs = sparse.hstack(Xs ).tocsr()
else:
Xs = self.merge_dataframes_by_column(Xs)
 return Xs
class PandasPipeline(TransformerMixin, BaseEstimator):
def __init__(self, steps=None):
self.steps = steps
if self.steps is None:
self.steps = []
def fit_transform(self, X, y=None, **fit_kwargs):
X = X.copy()
for name,trs in self.steps:
X = trs.fit_transform(X)
 return X
def transform(self, X):
X = X.copy()
for name,trs in self.steps:
X = trs.transform(X)
 return X
class FilterCommonValues(TransformerMixin, BaseEstimator):
def __init__(self, train, test, cols, lim=5, fill_val=-1):
self.train = train
self.test = test
self.cols = cols
self.lim = lim
self.fill_val = fill_val
def transform(self, X):
for col in self.cols:
if col in X:
X.loc[~X[col].isin(self.keep_vals_[col]), col] = self.fill_val
return X
def fit_transform(self, X, y=None, **fit_kwargs):
self.keep_vals_ = {}
for col in self.cols:
lim_vals =(self.train[col].value_counts() < self.lim ).replace(False, np.nan ).dropna().index.tolist()
common_vals = list(set(self.test[col].unique() ).intersection(self.train[col].unique()))
self.keep_vals_[col] = list(set(lim_vals + common_vals))
return self.transform(X)
 class SelectColumns(TransformerMixin, BaseEstimator):
def __init__(self, items=None, like=None, regex=None):
self.items = items
self.like = like
self.regex = regex
def fit_transform(self, X, y=None, **fit_params):
return self.transform(X)
 def transform(self, X):
X = X.filter(items=self.items, like=self.like,
regex=self.regex, axis=1)
 if X.shape[1] == 0:
logger.info('Cannot find anything with search %s, %s, %s' %(
 self.items,self.like,self.regex))
if VERBOSE > 1:
msg = X.columns.values if X.shape[1] < 50 else np.random.choice(X.columns.values.flatten() , size=50)
 logger.info(f'Selected {X.shape[1]} columns(max 50 shown): {msg}')
 return X
class DropColumns(TransformerMixin, BaseEstimator):
def __init__(self, items=None, like=None, regex=None):
self.items = items
self.like = like
self.regex = regex
def fit_transform(self, X, y=None, **fit_params):
return self.transform(X)
 def transform(self, X):
drop_cols = X.iloc[0:0].filter(items=self.items, like=self.like,
regex=self.regex, axis=1 ).columns.tolist()
X = X.drop(drop_cols, axis=1)
 if VERBOSE > 1:
msg = X.columns.values if X.shape[1] < 100 else X.shape[1]
logger.info(f'Dropped to {X.shape[1]} columns: {msg}')
 return X
class ConvertType(TransformerMixin, BaseEstimator):
def __init__(self, cols, dtype):
self.cols = cols
self.dtype = dtype
def fit_transform(self, X, y=None, **fit_params):
return self.transform(X)
 def transform(self, X):
X[self.cols] = X[self.cols].astype(self.dtype)
 return X
class SelectType(TransformerMixin, BaseEstimator):
def __init__(self, include=None, exclude=None):
self.include = include
self.exclude = exclude
def fit_transform(self, X, y=None, **fit_kwargs):
return self.transform(X)
 def transform(self, X):
return X.select_dtypes(include=self.include, exclude=self.exclude )
img_size = 256
batch_size = 256<define_search_model>
class StemmedTfidfVectorizer(TfidfVectorizer):
def __init__(self, stemmer, *args, **kwargs):
super(StemmedTfidfVectorizer, self ).__init__(*args, **kwargs)
 self.stemmer = stemmer
def build_analyzer(self):
analyzer = super(StemmedTfidfVectorizer, self ).build_analyzer()
return lambda doc:(self.stemmer.stem(word)for word in analyzer(doc.replace('
', ' ')))
 def grouped_feature(X, gr_col, agg_cols, agg_funcs):
gr = X.groupby(gr_col)[agg_cols].agg(agg_funcs)
 gr.columns = [l1+'_'+'_'.join(gr_col)+'_'+l2 for l1,l2 in zip(gr.columns.get_level_values(0),
gr.columns.get_level_values(1)) ]
gr = gr.reset_index()
X = X.merge(gr, how='left', on=gr_col)
 return X[gr.columns.drop(gr_col)]
class StratifiedBagger(BaseEstimator,RegressorMixin):

 def __init__(self, clf=None, n_bags=10, random_state=None, labelize=False, early_stopping_rounds=None):
self.clf = clf
self.n_bags = n_bags
self.random_state = random_state if random_state is not None else SEED
self.labelize = labelize
self.early_stopping_rounds = early_stopping_rounds
def fit(self, X, y=None, **fit_params):
skf = StratifiedKFold(self.n_bags, shuffle=True, random_state=self.random_state)
 folds = skf.split(X, y)
 self.clf_ = []
self.y_ = y
for i,(train_idx,valid_idx)in enumerate(folds):
if isinstance(X, pd.DataFrame): 
(X_train,y_train,X_valid,y_valid)=(X.iloc[train_idx],y.iloc[train_idx],
X.iloc[valid_idx],y.iloc[valid_idx])
 else: 
(X_train,y_train,X_valid,y_valid)=(X[train_idx],y.iloc[train_idx],
X[valid_idx],y.iloc[valid_idx])
 clf = deepcopy(self.clf)
 self.clf_.append(clf)
 if self.early_stopping_rounds is None:
self.clf_[i].fit(X_train, y_train, **fit_params)
 else:
self.clf_[i].fit(X_train, y_train, eval_set=[(X_valid,y_valid)],
early_stopping_rounds=self.early_stopping_rounds,
verbose=False, **fit_params)
 if VERBOSE:
if isinstance(self.clf_[i], LGBMRegressor):
fig,ax = plt.subplots(1, 1, figsize=(12,12))
plot_importance(self.clf_[i], ax=ax)
 plt.show()
if isinstance(self.clf_[i], XGBRegressor):
fig,ax = plt.subplots(1, 1, figsize=(12,12))
xgb_plot_importance(self.clf_[i], ax=ax)
 plt.show()
if isinstance(self.clf_[i], CatBoostRegressor):
fea_imp = pd.DataFrame({'imp': self.clf_[i].feature_importances_,
'col': X_train.columns})
 fea_imp = fea_imp.sort_values(['imp', 'col'], ascending=[True, False])
 fea_imp.plot(kind='barh', x='col', y='imp', figsize=(12,12))
plt.show()
return self
def predict(self, X, y=None):
y_preds = np.zeros(( X.shape[0],self.n_bags))
for i in range(self.n_bags):
y_pred = minmax_scale(self.clf_[i].predict(X),(0,4))
if self.labelize:
y_pred = labelize_match_target(y_pred, self.y_)
 y_preds[:,i] = y_pred
if self.labelize:
y_preds = mode(y_preds.T)[0].T.flatten().astype(np.int32)
 else:
y_preds = y_preds.mean(-1)
 return y_preds
class OOFTransformer(TransformerMixin, BaseEstimator):

 def __init__(self, clf, n_folds=5, agg_func='mean', name=None):
self.clf = clf
self.n_folds = n_folds
self.agg_func = agg_func
self.name = name
if self.name is None:
self.name = str(self.clf.__class__ ).split('.')[-1][1:-2]
def fit_transform(self, X, y, **fit_kwargs):
self.name_ = 'oof_' + self.name
self.clf_ = []
skf = StratifiedKFold(self.n_folds, shuffle=True, random_state=SEED+1234)
 folds = skf.split(X, y)
 X_train = np.zeros(( X.shape[0],))
if not SUBMISSION and not USE_LB:
path = f'{self.name_}_{CV}_{FOLD}_X_train.csv'
if os.path.exists(path)and os.path.exists(path.replace('X_train','X_test')) :
if VERBOSE:
logger.info(f'Read existing for {self.name_}(fit)')
 return pd.read_csv(path)
 for i,(fold_idx,oof_idx)in enumerate(folds):
self.clf_.append(deepcopy(self.clf))
if VERBOSE:
logger.info(f'Generating {self.name_} oof {i+1}/{self.n_folds}')
 if isinstance(X, pd.DataFrame): 
(X_fold,y_fold,X_oof,y_oof)=(X.iloc[fold_idx],y.iloc[fold_idx],
X.iloc[oof_idx],y.iloc[oof_idx])
 else: 
(X_fold,y_fold,X_oof,y_oof)=(X[fold_idx],y.iloc[fold_idx],
X[oof_idx],y.iloc[oof_idx])
 self.clf_[i].fit(X_fold, y_fold)
 X_train[oof_idx] = self.clf_[i].predict(X_oof)
 if VERBOSE:
if isinstance(self.clf_[i], LGBMRegressor):
fig,ax = plt.subplots(1, 1, figsize=(18,18))
plot_importance(self.clf_[i], ax=ax)
 plt.show()
if isinstance(self.clf_[i], XGBRegressor):
fig,ax = plt.subplots(1, 1, figsize=(18,18))
xgb_plot_importance(self.clf_[i], ax=ax)
 plt.show()
if isinstance(self.clf_[i], CatBoostRegressor):
fea_imp = pd.DataFrame({'imp': self.clf_[i].feature_importances_,
'col': X_train.columns})
 fea_imp = fea_imp.sort_values(['imp', 'col'], ascending=[True, False] ).iloc[-30:]
fea_imp.plot(kind='barh', x='col', y='imp', figsize=(18,18))
plt.show()
X_train = pd.DataFrame(X_train, columns=[self.name_])
 if not SUBMISSION and not USE_LB:
X_train.to_csv(path, index=False)
 return X_train
def transform(self, X):
if not SUBMISSION and not USE_LB:
path = f'{self.name_}_{CV}_{FOLD}_X_test.csv'
if os.path.exists(path):
if VERBOSE:
print(f'Read existing for {self.name_}(transform)')
 return pd.read_csv(path)
 X_test = np.zeros(( X.shape[0],self.n_folds))
for i in range(self.n_folds):
X_test[:,i] = self.clf_[i].predict(X)
 if self.agg_func == 'mean':
X_test = X_test.mean(-1)
 elif self.agg_func == 'median':
X_test = np.median(X_test, axis=-1)
 else:
raise ValueError(f'Unknown aggregation {self.agg_func}')
 X_test = pd.DataFrame(X_test, columns=[self.name_])
 if not SUBMISSION and not USE_LB:
X_test.to_csv(path, index=False)
 return X_test
class PredictTransformer(BaseEstimator, TransformerMixin):
def __init__(self, clf, labelize=True, name=None):
self.clf = clf
self.labelize = labelize
self.name = name
if self.name is None:
self.name = str(self.clf.__class__ ).split('.')[-1][1:-2]
def fit(self, X, y, **fit_kwargs):
self.name_ = 'pred_' + self.name
self.clf.fit(X, y, **fit_kwargs)
 if VERBOSE:
print(f'Fitting {self.name_}...')
 if isinstance(self.clf, LGBMRegressor):
fig,ax = plt.subplots(1, 1, figsize=(18,18))
plot_importance(self.clf, ax=ax)
 plt.show()
if isinstance(self.clf, XGBRegressor):
fig,ax = plt.subplots(1, 1, figsize=(18,18))
xgb_plot_importance(self.clf, ax=ax)
 plt.show()
if isinstance(self.clf, CatBoostRegressor):
fea_imp = pd.DataFrame({'imp': self.clf.feature_importances_,
'col': X_train.columns})
 fea_imp = fea_imp.sort_values(['imp', 'col'], ascending=[True, False] ).iloc[-30:]
fea_imp.plot(kind='barh', x='col', y='imp', figsize=(18,18))
plt.show()
y_pred = self.clf.predict(X)
 if self.labelize:
self.y_ = y
return self
def transform(self, X):
y_pred = self.clf.predict(X)
 if self.labelize:
y_pred = labelize_match_target(y_pred, self.y_)
 return pd.DataFrame(y_pred, columns=[self.name_] )
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights=".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )<define_variables>
breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 color = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 state = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )
pet_ids = train['PetID'].values
n_batches = int(np.ceil(len(pet_ids)/ batch_size))
features = {}
for b in tqdm_notebook(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]<create_dataframe>
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 logger.info('Train shape: %s' % str(train.shape))
logger.info('Test shape: %s' % str(test.shape))
train_feats = pd.DataFrame.from_dict(features, orient='index')
 train_feats.columns = [f'pic_{i}' for i in range(train_feats.shape[1])]<predict_on_test>
ext_state = pd.read_excel('.. /input/malaysian-state-data/external_state_data.xlsx',
decimal=',')[['State','Population2','2016 GDP']]
train = train.merge(ext_state, how='left', on='State')
 test = test.merge(ext_state, how='left', on='State' )
pet_ids = test['PetID'].values
n_batches = int(np.ceil(len(pet_ids)/ batch_size))
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/test_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]<create_dataframe>
train['Description'] = train['Description'].fillna('' ).astype(str)
 test['Description'] = test['Description'].fillna('' ).astype(str )
test_feats = pd.DataFrame.from_dict(features, orient='index')
 test_feats.columns = [f'pic_{i}' for i in range(test_feats.shape[1])]<rename_columns>
def create_metadata(pet_id, train=True):
all_res = {}
for pic in range(1,2):
meta_path = '.. /input/petfinder-adoption-prediction/%s_metadata/%s-%s.json' %('train' if train else 'test', pet_id,pic)
 pic_path = '.. /input/petfinder-adoption-prediction/%s_images/%s-%s.jpg' %('train' if train else 'test', pet_id,pic)
 res = {
'PetID': pet_id,
f'vertex_y{pic}': 0,
f'label_score{pic}': 0,
}
try:
if os.path.exists(meta_path):
with open(meta_path, 'r')as f:
d = json.load(f)
 res[f'vertex_y{pic}'] = d['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
if 'labelAnnotations' in d:
res[f'label_score{pic}'] = d['labelAnnotations'][0]['score'] + 1
all_res.update(res)
 except:
logger.warning('Failed loading JSON files!')
 pass
return all_res
train_feats = train_feats.reset_index()
train_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 test_feats = test_feats.reset_index()
test_feats.rename({'index': 'PetID'}, axis='columns', inplace=True )<concatenate>
logger.info('Creating metadata features...')
 train_metadata_feats = Parallel(n_jobs=4 )(delayed(create_metadata )(pet_id)
 for pet_id in tqdm_notebook(train['PetID'].unique()))
train_metadata_feats = pd.DataFrame(train_metadata_feats)
 create_metadata_test = partial(create_metadata, train=False)
 test_metadata_feats = Parallel(n_jobs=4 )(delayed(create_metadata_test )(pet_id)
 for pet_id in tqdm_notebook(test['PetID'].unique()))
test_metadata_feats = pd.DataFrame(test_metadata_feats)
 logger.info('Merging metadata features...')
 train = train.merge(train_metadata_feats, how='left', on='PetID')
 test = test.merge(test_metadata_feats, how='left', on='PetID')
 del train_metadata_feats,test_metadata_feats
gc.collect()
logger.info('Metadata features done!' )
all_ids = pd.concat([train, test], axis=0, ignore_index=True, sort=False)[['PetID']]
all_ids.shape<concatenate>
logger.info('Creating image features...')
 train_img_feats,test_img_feats = create_features(normal, 'xception', 'avg')
 tmp_train,tmp_test = create_features(normal, 'nasnetlarge', 'avg')
 train_img_feats = train_img_feats.merge(tmp_train, how='left', on='PetID')
 test_img_feats = test_img_feats.merge(tmp_test, how='left', on='PetID')
 logger.info('Merging image features...')
 train = train.merge(train_img_feats, how='left', on='PetID')
 test = test.merge(test_img_feats, how='left', on='PetID')
 del train_img_feats,test_img_feats
gc.collect()
logger.info('Image features done!' )
n_components = 32
svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 features_df = pd.concat([train_feats, test_feats], axis=0)
 features = features_df[[f'pic_{i}' for i in range(256)]].values
svd_col = svd_.fit_transform(features)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('IMG_SVD_')
 img_features = pd.concat([all_ids, svd_col], axis=1 )<load_from_csv>
def row_features(X):

 X['adopted'] = 1
X.loc[X['Description'].str.lower().str.contains('adopted'),'adopted'] = 0
X.loc[X['Description'].str.lower().str.contains('adoption'),'adopted'] = 2
rescuer_counts =(
 X[['RescuerID','PetID']] 
.groupby('RescuerID' ).agg({'PetID': lambda x:(x.count() > 1 ).astype(np.int8)})
.rename(columns={'PetID': 'MoreThanOneRescuerPet'})
.reset_index() 
)
 X = X.merge(rescuer_counts, how='left', on='RescuerID')
 return X[['adopted','MoreThanOneRescuerPet']]
class ExtractLeaky(TransformerMixin, BaseEstimator):

 def __init__(self, train, test):
self.train = train
self.test = test
def _ratio(self, gr1, gr2, agg_col='PetID', agg_func='count'):
req_cols = list(set(gr1 + gr2 + [agg_col]))
df = pd.concat([self.train[req_cols], self.test[req_cols]], axis=0)
 df1 = df.groupby(gr1)[agg_col].agg(agg_func ).to_frame('gr1' ).reset_index()
df2 = df.groupby(gr2)[agg_col].agg(agg_func ).to_frame('gr2' ).reset_index()
df = df.merge(df1, how='left', on=gr1 ).merge(df2, how='left', on=gr2)
 colname = '%s_%s_%s_%s' %(agg_col,agg_func,'-'.join(gr1),'-'.join(gr2))
df[colname] =(df['gr1'] / df['gr2'] ).replace([-np.inf,np.inf],np.nan)
 df = df.drop(['gr1','gr2'], axis=1)
 self.train_feats_ = pd.concat([
self.train_feats_, df[colname].iloc[:self.n_train_]], axis=1)
 self.test_feats_ = pd.concat([
self.test_feats_, df[colname].iloc[self.n_train_:].reset_index(drop=True)], axis=1)
 def fit_transform(self, X, y=None, **fit_kwargs):
self.n_train_ = self.train.shape[0]
self.train_feats_ = pd.DataFrame()
self.test_feats_ = pd.DataFrame()
self._ratio(['Breed1'], ['State'], 'PetID', 'count')
 self._ratio(['Breed1','RescuerID'], ['PetID'], 'PetID', 'count')
 self._ratio(['PetID'], ['Breed1'], 'PhotoAmt', 'mean')
 self._ratio(['RescuerID'], ['PetID'], 'Breed1', 'nunique')
 self.train_feats_['Breed1_nunique_RescuerID_PetID'] =(
 self.train_feats_['Breed1_nunique_RescuerID_PetID'].clip(1, self.test_feats_['Breed1_nunique_RescuerID_PetID'].max()))
self.test_feats_['Breed1_nunique_RescuerID_PetID'] =(
 self.test_feats_['Breed1_nunique_RescuerID_PetID'].clip(1, self.test_feats_['Breed1_nunique_RescuerID_PetID'].max()))
return self.train_feats_
def transform(self, X):
return self.test_feats_
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )<define_variables>
target = 'AdoptionSpeed'
common_cols = test.columns.intersection(train.columns ).values.tolist()
train = train[common_cols + [target]]
test = test[common_cols]
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
if debug:
train_image_files = train_image_files[:1000]
train_metadata_files = train_metadata_files[:1000]
train_sentiment_files = train_sentiment_files[:1000]
print(f'num of train images files: {len(train_image_files)}')
 print(f'num of train metadata files: {len(train_metadata_files)}')
 print(f'num of train sentiment files: {len(train_sentiment_files)}')
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print(f'num of test images files: {len(test_image_files)}')
 print(f'num of test metadata files: {len(test_metadata_files)}')
 print(f'num of test sentiment files: {len(test_sentiment_files)}' )<load_pretrained>
rescuer_ids = get_rescuer_ids(train )
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_json_file(self, filename):
with open(filename, 'r', encoding='utf-8')as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns')
 file_sentences_sentiment_df = pd.DataFrame(
 {
'magnitude_sum': file_sentences_sentiment['magnitude'].sum(axis=0),
'score_sum': file_sentences_sentiment['score'].sum(axis=0),
'magnitude_mean': file_sentences_sentiment['magnitude'].mean(axis=0),
'score_mean': file_sentences_sentiment['score'].mean(axis=0),
'magnitude_var': file_sentences_sentiment['magnitude'].var(axis=0),
'score_var': file_sentences_sentiment['score'].var(axis=0),
}, index=[0] 
)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
df_sentiment = pd.concat([df_sentiment, file_sentences_sentiment_df], axis=1)
 df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = f'.. /input/petfinder-adoption-prediction/{mode}_sentiment/{pet_id}.json'
try:
sentiment_file = pet_parser.open_json_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
for ind in range(1,200):
metadata_filename = '.. /input/petfinder-adoption-prediction/{}_metadata/{}-{}.json'.format(mode, pet_id, ind)
 try:
metadata_file = pet_parser.open_json_file(metadata_filename)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 except FileNotFoundError:
break
if dfs_metadata:
dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()<count_unique_values>
if LB:
train_idx,test_idx = train_test_split(rescuer_ids['unique'],
stratify=rescuer_ids['count'], test_size=0.2, random_state=209321206+73)
 train_idx = [idx for sidx in train_idx for idx in sidx]
test_idx = [idx for sidx in test_idx for idx in sidx]
X_train,y_train,X_test,y_test =(train.iloc[train_idx].drop('AdoptionSpeed', axis=1),
train.iloc[train_idx][target],
train.iloc[test_idx].drop('AdoptionSpeed', axis=1),
train.iloc[test_idx][target])
 rescuer_ids = get_rescuer_ids(train.iloc[train_idx])
 else:
X_train,y_train,X_test = train.drop('AdoptionSpeed', axis=1),train[target],test
<categorify><EOS> 
def run_level0(X_train, y_train, X_test):
text_feats = PandasFeatureUnion([ 
('Original', PandasTransform(lambda x: x)) , 
('Raw description + TFIDF + SVD10', PandasPipeline([ 
('select', SelectColumns(['Description'])) , 
('preprocess', PandasTransform(lambda x: x.values.flatten().tolist())) , 
('tfidf', TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)) , 
('svd', TruncatedSVD(n_components=10, random_state=1337)) , 
('df', PandasTransform(lambda x:
pd.DataFrame(x, columns=[f'lvl0_text_raw_desc_tfidf_svd10_{i}' for i in range(10)])))
 ])) , 
('Stemmed description + TFIDF + SVD5', PandasPipeline([ 
('select', SelectColumns(['Description'])) , 
('preprocess', PandasTransform(lambda x: x.fillna('' ).values.flatten().tolist())) , 
('tfidf', StemmedTfidfVectorizer(SnowballStemmer('english', ignore_stopwords=False),
min_df=2, max_features=None, strip_accents='unicode', analyzer='word',
token_pattern=r'(?u)\b\w+\b', ngram_range=(1, 3), use_idf=1, smooth_idf=1,
sublinear_tf=1)) , 
('svd', TruncatedSVD(n_components=5, random_state=1337)) , 
('df', PandasTransform(lambda x:
pd.DataFrame(x, columns=[f'lvl0_text_stem_desc_tfidf_svd5_{i}' for i in range(5)])))
 ]))
], n_jobs=1)
 X_train = text_feats.fit_transform(X_train, y_train)
 X_test = text_feats.transform(X_test)
 img_feats = PandasFeatureUnion([ 
('Original', PandasTransform(lambda x: x)) , 
('NasNet + PCA5', PandasPipeline([ 
('select cols', SelectColumns(regex=r'(nasnetlarge_)')) , 
('impute', SimpleImputer(fill_value=0.)), 
('var_thres', VarianceThreshold(0.)), 
('pca', PCA(5, random_state=234)) , 
('df', PandasTransform(lambda x:
pd.DataFrame(x, columns=[f'lvl0_img_nasnet_pca5_{i}' for i in range(5)])))
 ])) , 
('NasNet + PCA400 + LR', OOFTransformer(make_pipeline(
 SelectColumns(regex=r'(nasnetlarge_)'),
SimpleImputer(fill_value=0.) ,
VarianceThreshold(0.) ,
PCA(400, random_state=234),
LinearRegression(
)
), name='lvl0_img_nasnet_pca400_lr')) , 
('Xception + SVD150 + ET', OOFTransformer(make_pipeline(
 SelectColumns(regex=r'(xception_)'),
SimpleImputer(fill_value=0.) ,
VarianceThreshold(0.) ,
TruncatedSVD(150, random_state=113120),
ExtraTreesRegressor(
 n_estimators=200,
random_state=10233 
)
), name='lvl0_img_xception_svd150_pca400_lr')) ,
], n_jobs=1)
 X_train = img_feats.fit_transform(X_train, y_train)
 X_test = img_feats.transform(X_test)
 if SUBMISSION or VERBOSE:
display(X_train.filter(like='oof_lvl0' ).corr())
 display(X_test.filter(like='oof_lvl0' ).corr())
 if SUBMISSION:
X_test.filter(like='oof_lvl0' ).to_csv('X_test_lvl0.csv', index=False)
 return X_train,X_test
def run_level1(X_train, y_train, X_test):
normal_feats = [
'Type', 'Age', 'Breed1', 'Breed2', 'Gender',
'Color1', 'Color2', 'Color3', 'MaturitySize',
'FurLength', 'Vaccinated', 'Dewormed', 'Sterilized',
'Health', 'Quantity', 'Fee', 'State', 'VideoAmt', 'PhotoAmt',
'Population2', '2016 GDP', 'adopted', 'MoreThanOneRescuerPet',
]
lvl1 = PandasFeatureUnion([ 
('Original', PandasTransform(lambda x: x)) , 
('LGB + Non-image', OOFTransformer(make_pipeline(
 SelectColumns(normal_feats),
LGBMRegressor(
 n_estimators=100,
num_leaves=16,
learning_rate=0.1,
max_depth=5,
colsample_bytree=0.5,
min_split_gain=0.02,
min_child_samples=100,
min_child_weight=0.02,
reg_lambda=0.01,
random_state=SEED+374,
objective='rmse' 
)
), name='lvl1_lgb_non_img')) , 
('LGB + Image and text', OOFTransformer(make_pipeline(
 SelectColumns(regex=r'(lvl0_text|lvl0_img|PhotoAmt|label_score|vertex_y)'),
LGBMRegressor(
 n_estimators=100,
num_leaves=16,
learning_rate=0.1,
max_depth=5,
colsample_bytree=0.5,
min_split_gain=0.02,
min_child_samples=100,
min_child_weight=0.02,
reg_lambda=0.01,
random_state=SEED+274,
objective='rmse' 
)
), name='lvl1_lgb_img_text')) , 
('LGB + All features bagging', OOFTransformer(make_pipeline(
 SelectColumns(regex=r'(lvl0|label_score|vertex_y|%s)' % '|'.join(normal_feats)) ,
BaggingRegressor(
 LGBMRegressor(
 n_estimators=100,
num_leaves=16,
learning_rate=0.1,
max_depth=5,
colsample_bytree=0.5,
min_split_gain=0.02,
min_child_samples=100,
min_child_weight=0.02,
reg_lambda=0.01,
random_state=SEED+743,
objective='rmse' 
),
n_estimators=20,
max_samples=0.8,
max_features=0.4,
bootstrap=True,
random_state=9843 
)
), name='lvl1_lgb_bagging'))
], n_jobs=1)
 X_train = lvl1.fit_transform(X_train, y_train)
 X_test = lvl1.transform(X_test)
 lvl1_rescuer_oof_mean = PandasFeatureUnion([ 
('original', PandasTransform(lambda x: x)) , 
('rescuermean', PandasTransform(lambda x:
grouped_feature(x, ['RescuerID'], ['oof_lvl1_lgb_bagging'], ['mean'])))
 ])
 X_train = lvl1_rescuer_oof_mean.fit_transform(X_train)
 X_test = lvl1_rescuer_oof_mean.transform(X_test)
 if SUBMISSION or VERBOSE:
display(X_train.filter(regex='(oof_lvl1_|pred_)' ).corr())
 display(X_test.filter(regex='(oof_lvl1_|pred_)' ).corr())
 if SUBMISSION:
X_test.filter(regex='(oof_lvl1_|pred_)' ).to_csv('X_test_lvl1.csv', index=False)
 return X_train,X_test
def run_level2(X_train, y_train, X_test):
normal_feats = [
'Type', 'Age', 'Breed1', 'Breed2', 'Gender',
'Color1', 'Color2', 'Color3', 'MaturitySize',
'FurLength', 'Vaccinated', 'Dewormed', 'Sterilized',
'Health', 'Quantity', 'Fee', 'State', 'VideoAmt', 'PhotoAmt',
'Population2', '2016 GDP', 'adopted', 'MoreThanOneRescuerPet',
]
lvl2 = PandasFeatureUnion([ 
('Best LGB', PredictTransformer(make_pipeline(
 SelectColumns(regex=r'(lvl0|lvl1|label_score|vertex_y|%s)' % '|'.join(normal_feats)) ,
LGBMRegressor(
 n_estimators=100,
num_leaves=16,
learning_rate=0.1,
max_depth=5,
colsample_bytree=0.5,
min_split_gain=0.02,
min_child_samples=100,
min_child_weight=0.02,
reg_lambda=0.01,
random_state=SEED+74,
objective='rmse' 
)
), name='lvl2_lgb')) , 
('Best CatBoost', PredictTransformer(make_pipeline(
 SelectColumns(regex=r'(lvl0|lvl1|label_score|vertex_y|%s)' % '|'.join(normal_feats)) ,
CatBoostRegressor(
 n_estimators=250,
learning_rate=0.1,
max_depth=6,
colsample_bylevel=0.5,
l2_leaf_reg=1.,
random_state=SEED+1932,
verbose=-1 
)
), name='lvl2_catboost')) , 
('Best Ridge', PredictTransformer(make_pipeline(
 SelectColumns(regex=r'(oof_lvl1_lgb)'),
DropColumns(regex=r'(RescuerID)'),
BaggingRegressor(
 BayesianRidge(
 normalize=False,
fit_intercept=True 
),
n_estimators=100,
max_samples=0.8,
max_features=1.0,
bootstrap=True,
random_state=93843 
)
), name='lvl2_ridge'))
], n_jobs=1)
 X_train = lvl2.fit_transform(X_train, y_train)
 X_test = lvl2.transform(X_test)
 if SUBMISSION or VERBOSE:
display(X_train.filter(regex='(oof_lvl2_|pred_)' ).corr())
 display(X_test.filter(regex='(oof_lvl2_|pred_)' ).corr())
 if SUBMISSION:
X_test.filter(regex='(oof_lvl2_|pred_)' ).to_csv('X_test_lvl2.csv', index=False)
 return X_train,X_test
def run_level3(X_train, y_train, X_test):
y_pred = mode(X_test.values.T)[0].T.flatten().astype(np.int32)
 return y_pred
def run_all(X_train, y_train, X_test):
X_train = X_train.reset_index(drop=True ).copy()
X_test = X_test.reset_index(drop=True ).copy()
if VERBOSE or SUBMISSION:
logger.info('>> Fitting basic pipeline...')
 pl = PandasPipeline([ 
('extract_features', PandasFeatureUnion([ 
('Original features', PandasTransform(lambda x: x)) , 
('Row features', PandasTransform(row_features)) , 
('Leaky features', ExtractLeaky(X_train, X_test))
])) , 
('Common values', FilterCommonValues(X_train, X_test, ['Breed1','Breed2','State'])) , 
('Categoricals', ConvertType([
'Type','Breed1','Breed2','Gender','Color1','Color2','Color3','MaturitySize',
'FurLength','Vaccinated','Dewormed','Sterilized','Health','State','adopted'],
'category'))
])
 X_train = pl.fit_transform(X_train)
 X_test = pl.transform(X_test)
 use_feats =(X_train.iloc[0:0].select_dtypes(include=['category','number'])
.columns.tolist() + ['RescuerID','Description'])
 X_train,X_test = X_train[use_feats],X_test[use_feats]
if VERBOSE or SUBMISSION:
logger.info('>> Level 0...')
 X_train,X_test = run_level0(X_train, y_train, X_test)
 if VERBOSE:
logger.info('>> Level 1...')
 X_train,X_test = run_level1(X_train, y_train, X_test)
 if VERBOSE:
logger.info('>> Level 2...')
 X_train,X_test = run_level2(X_train, y_train, X_test)
 if VERBOSE:
logger.info('>> Level 3...')
 y_pred = run_level3(X_train, y_train, X_test)
 if VERBOSE or SUBMISSION:
display(y_pred)
 return y_pred
USE_LB = False
if not SUBMISSION:
validate(X_train, y_train, rescuer_ids)
 if LB:
USE_LB = True
y_pred = run_all(X_train, y_train, X_test)
 msg = 'LB score: %.3f' % qwk(y_pred, y_test)
 logger.info(msg)
 if SUBMISSION and not LB:
y_pred = pd.DataFrame(run_all(X_train, y_train, X_test),
columns=['AdoptionSpeed'],
index=test['PetID'] ).reset_index()
y_pred.to_csv('submission.csv', index=False)
 fig,ax = plt.subplots(1,2,figsize=(16,4))
y_pred['AdoptionSpeed'].hist(ax=ax[0])
 train['AdoptionSpeed'].hist(ax=ax[1])
 ax[0].set_title('Prediction distribution')
 ax[1].set_title('Train set distribution')
 else:
logger.info('Will not make submission without full dataset, turn LB off!' )