Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<compute_test_metric>
%matplotlib inline
plt.rc('figure', figsize=(20.0, 10.0))
print('Mean Absolute Error:', metrics.mean_absolute_error(y_test, y_pred_lgb))
print('Mean Squared Error:', metrics.mean_squared_error(y_test, y_pred_lgb))
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test, y_pred_lgb)) )<compute_test_metric>
train_df = pd.read_csv(os.path.join(INPUT_DIR, 'train', 'train.csv'))
X_test = pd.read_csv(os.path.join(INPUT_DIR, 'test', 'test.csv'))
preds_ensemble_avg =(y_pred_xgb + y_pred_lgb)/2<compute_test_metric>
class DataFrameColumnMapper(BaseEstimator, TransformerMixin):

 def __init__(self, column_name, mapping_func, new_column_name=None, drop_original=True):

 self.column_name = column_name
self.mapping_func = mapping_func
self.new_column_name = new_column_name if new_column_name is not None else self.column_name
self.drop_original = drop_original
def fit(self, X, y=None):
return self
def transform(self, X):
transformed_column = X.transform({self.column_name: self.mapping_func})
 Y = X.copy()
Y = Y.assign(**{self.new_column_name: transformed_column})
 if self.column_name != self.new_column_name and self.drop_original:
Y = Y.drop(self.column_name, axis=1)
 return Y
print("Averaging Ensemble predictions KPI here:")
 print('Mean Absolute Error:', metrics.mean_absolute_error(y_test, preds_ensemble_avg))
print('Mean Squared Error:', metrics.mean_squared_error(y_test, preds_ensemble_avg))
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test, preds_ensemble_avg)) )<define_variables>
class CategoricalToOneHotEncoder(BaseEstimator, TransformerMixin):

 def __init__(self, columns=None):
self.columns = columns
self.mappings_ = None
def fit(self, X, y=None):
if self.columns is None:
self.columns = X.select_dtypes(exclude='number')
 mappings = {}
for col in self.columns:
labels, uniques = X.loc[:, col].factorize()
int_and_cat = list(enumerate(uniques))
cat_and_int = [(x[1], x[0])for x in int_and_cat]
mappings[col] = {'int_to_cat': dict(int_and_cat), 'cat_to_int': dict(cat_and_int)}
self.mappings_ = mappings
return self
def transform(self, X):
Y = X.copy()
for col in self.columns:
transformed_col = Y.loc[:, col].transform(lambda x: self.mappings_[col]['cat_to_int'][x])
 for key, val in self.mappings_[col]['cat_to_int'].items() :
one_hot =(transformed_col == val)+ 0
Y = Y.assign(**{'{}_{}'.format(col, key): one_hot})
 Y = Y.drop(col, axis=1)
 return Y
preds_ensemble_avg =(y_pred_xgb*0.1 + y_pred_lgb *0.9 )<compute_test_metric>
class CategoricalTruncator(BaseEstimator, TransformerMixin):

 def __init__(self, column_name, n_values_to_keep=5):
self.column_name = column_name
self.n_values_to_keep = n_values_to_keep
self.values_ = None
def fit(self, X, y=None):
self.values_ = list(X[self.column_name].value_counts() [:self.n_values_to_keep].keys())
 return self
def transform(self, X):
transform = lambda x: x if x in self.values_ else 'Other'
Y = X.copy()
y = Y.transform({self.column_name: transform})
 return Y.assign(**{self.column_name: y} )
print("Weighted average Ensemble predictions KPI here:")
 print('Mean Absolute Error:', metrics.mean_absolute_error(y_test, preds_ensemble_avg))
print('Mean Squared Error:', metrics.mean_squared_error(y_test, preds_ensemble_avg))
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test, preds_ensemble_avg)) )<predict_on_test>
class DataFrameColumnDropper(BaseEstimator, TransformerMixin):

 def __init__(self, column_names):
self.column_names = column_names
def fit(self, X, y=None):
return self
def transform(self, X):
return X.copy().drop(self.column_names, axis=1 )
preds_xgb = XGB.predict(df_test.iloc[:,1:].values )<predict_on_test>
class ColumnByFeatureImportancePicker(BaseEstimator, TransformerMixin):

 def __init__(self, n_features: int = 20, classifier=RandomForestClassifier(n_estimators=100, random_state=42)) :
self.n_features = n_features
self.classifier = classifier
self.attributes_ = None
def fit_and_compute_importances(self, X_df, y):

 X_numeric = X_df.select_dtypes(include='number')
 X = X_numeric.values
self.classifier.fit(X, y)
 feature_importances = self.classifier.feature_importances_
feature_names = list(X_numeric)
 feature_importances_with_names = [(feature_name, feature_importance)for feature_name, feature_importance in zip(feature_names, feature_importances)]
feature_importances_with_names.sort(key=lambda x: x[1], reverse=True)
 return feature_importances_with_names
def fit(self, X, y=None):
if self.n_features is None:
self.attributes_ = list(X)
 return self
assert y is not None, "Feature importances cannot be computed without y!"
feature_importances_with_names = self.fit_and_compute_importances(X, y)
 self.attributes_ = [feature_name for feature_name, _ in feature_importances_with_names[:self.n_features]]
return self
def transform(self, X):
return X.copy().loc[:, self.attributes_]
preds_lgb = best_model.predict(df_test.iloc[:,1:].values )<load_from_csv>
class DataFrameToValuesTransformer(BaseEstimator, TransformerMixin):

 def __init__(self):
self.attributes_ = None
pass
def fit(self, X, y=None):
self.attributes_ = list(X)
 return self
def transform(self, X):
return X.loc[:, self.attributes_].values
sub=pd.read_csv(".. /input/tabular-playground-series-jan-2021/sample_submission.csv" )<save_to_csv>
def to_features_and_labels(df):
y = df['AdoptionSpeed'].values
X = df.drop('AdoptionSpeed', axis=1)
 return X, y
X_train_val, y_train_val = to_features_and_labels(train_df)
 X_train, X_val, y_train, y_val = train_test_split(X_train_val, y_train_val, test_size=0.20, random_state=42,
stratify=y_train_val)
 print("Shape of X_train:", X_train.shape)
 print("Shape of X_val:", X_val.shape)
 print("Shape of y_train:", y_train.shape)
 print("Shape of y_val:", y_val.shape )
sub.target =(preds_xgb*0.1+ preds_lgb*0.9)
 sub.to_csv("submission.csv", index=False )<import_modules>
def has_field_transformer(column_name, new_column_name=None, is_missing_func=pd.notna)-> TransformerMixin:
return DataFrameColumnMapper(column_name=column_name,
mapping_func=lambda name: np.int(is_missing_func(name)) ,
drop_original=True,
new_column_name=new_column_name if new_column_name is not None else column_name)
 def value_matches_transformer(column_name, new_column_name=None, matches=pd.notna)-> TransformerMixin:
return DataFrameColumnMapper(column_name=column_name,
mapping_func=lambda value: np.int(matches(value)) ,
drop_original=False,
new_column_name=new_column_name if new_column_name is not None else column_name)
 def map_categories(column_name, mapping_dict)-> TransformerMixin:
return DataFrameColumnMapper(column_name=column_name,
mapping_func=lambda x: mapping_dict[x])
 def onehot_encode(columns)-> TransformerMixin:
return CategoricalToOneHotEncoder(columns=columns)
 def truncate_categorical(column_name, n_values_to_keep=10):
return CategoricalTruncator(column_name=column_name, n_values_to_keep=n_values_to_keep)
 ONEHOT_ENCODED_COLUMNS = ["Type", "Breed1", "Breed2", "Gender", "Color1", "Color2", "Color3", "Health",
"FurLength", "Vaccinated", "Dewormed", "Sterilized", "State", "RescuerID"]
def build_preprocessing_pipeline() -> Pipeline:
return Pipeline([ 
('add_has_name', has_field_transformer(column_name="Name", new_column_name="hasName")) , 
('add_is_free', value_matches_transformer(column_name="Fee", new_column_name="isFree",
matches=lambda value: value < 1)) , 
('map_type_to_species', map_categories(column_name="Type", mapping_dict={1: 'dog', 2: 'cat'})) , 
('map_gender_to_names', map_categories(column_name="Gender", mapping_dict={1: 'male', 2: 'female', 3: 'mixed'})) , 
('truncate_breed1', truncate_categorical(column_name="Breed1", n_values_to_keep=10)) , 
('truncate_breed2', truncate_categorical(column_name="Breed2", n_values_to_keep=10)) , 
('truncate_state', truncate_categorical(column_name="State", n_values_to_keep=10)) , 
('truncate_rescuer_id', truncate_categorical(column_name="RescuerID", n_values_to_keep=10)) , 
('onehot_encode', CategoricalToOneHotEncoder(columns=ONEHOT_ENCODED_COLUMNS)) , 
('drop_unused_columns', DataFrameColumnDropper(
 column_names=['PetID', 'Description', 'Type_dog'
])) , 
('pick_columns_by_importance', ColumnByFeatureImportancePicker(n_features=None))
])
 preprocessing_pipeline = build_preprocessing_pipeline()
X_train_preprocessed = preprocessing_pipeline.fit_transform(X_train, y_train)
 X_val_preprocessed = preprocessing_pipeline.transform(X_val)
 X_train_preprocessed.head(10 )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import gc
import xgboost as xgb
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from hyperopt import hp, fmin, tpe, Trials
from hyperopt.pyll.base import scope
from tqdm import tqdm<load_from_csv>
print("Number of features:", len(list(X_train_preprocessed)))
 print("")
 print("Numerical columns:", list(X_train_preprocessed.select_dtypes(include="number")))
 print("")
 print("Non-numerical columns:", list(X_train_preprocessed.select_dtypes(exclude="number")) )
path = '.. /input/tabular-playground-series-jan-2021/'
train = pd.read_csv(path+'train.csv')
 train.set_index('id',drop=True,inplace=True)
 train.drop(284103,inplace=True )<define_variables>
def build_preparation_pipeline() :
return Pipeline([ 
('to_numpy', DataFrameToValuesTransformer()), 
('scaler', StandardScaler())
 ])
 def build_full_pipeline(classifier=None):
preprocessing_pipeline = build_preprocessing_pipeline()
preparation_pipeline = build_preparation_pipeline()
return Pipeline([ 
('preprocessing', preprocessing_pipeline), 
('preparation', preparation_pipeline), 
('classifier', classifier)
 ] )
features = [col for col in train.columns if 'cont' in col]
label = 'target'<set_options>
rf_classifier = RandomForestClassifier(n_estimators=100)
 rf_pipeline = build_full_pipeline(classifier=rf_classifier)
 cross_val_score(rf_pipeline, X_train, y_train, cv=5, scoring=make_scorer(cohen_kappa_score))
fig.clear()
plt.close(fig )<split>
def build_search(pipeline, param_distributions, n_iter=10):
return RandomizedSearchCV(pipeline, param_distributions=param_distributions,
cv=5, return_train_score=True, refit='cohen_kappa',
n_iter=n_iter,
scoring={
'accuracy': make_scorer(accuracy_score),
'cohen_kappa': make_scorer(cohen_kappa_score)
 },
verbose=1, random_state=42)
 def pretty_cv_results(cv_results,
sort_by='rank_test_cohen_kappa',
sort_ascending=True,
n_rows=20):
df = pd.DataFrame(cv_results)
 cols_of_interest = [key for key in df.keys() if key.startswith('param_')
 or key.startswith('mean_train')
 or key.startswith('mean_test_')
 or key.startswith('rank')]
return df.loc[:, cols_of_interest].sort_values(by=sort_by, ascending=sort_ascending ).head(n_rows)
 def run_search(search):
search.fit(X_train, y_train)
 print('Best score is:', search.best_score_)
 return pretty_cv_results(search.cv_results_ )
X_train,X_valid, y_train,y_valid = train_test_split(train[features],train[label],test_size=0.2)
 d_tr = xgb.DMatrix(X_train, y_train)
 d_val = xgb.DMatrix(X_valid,y_valid )<train_model>
param_distributions = {
'preprocessing__pick_columns_by_importance__n_features': [None, 50],
'classifier': [RandomForestClassifier(n_estimators=250, random_state=42, max_depth=10)],
'classifier__max_depth': [10, None]
}
rf_feature_search = build_search(build_full_pipeline() , param_distributions=param_distributions, n_iter=4)
 rf_feature_cv_results = run_search(search=rf_feature_search)
 rf_feature_cv_results
params_base = {'objective': 'reg:squarederror',
'tree_method': 'gpu_hist',
'random_state': 0}
base_model = xgb.train(params = params_base,
dtrain = d_tr,
num_boost_round = 1000,
evals = [(d_val,'eval')],
early_stopping_rounds=10,
verbose_eval = 20)
 y_pred_base = base_model.predict(d_val)
 base_score = mean_squared_error(y_valid, y_pred_base,squared=False)
 print(base_score )<choose_model_class>
param_distributions = {
'preprocessing__pick_columns_by_importance__n_features': [None],
'classifier': [RandomForestClassifier(n_estimators=500, random_state=42)],
'classifier__n_estimators': [500],
'classifier__max_features': ['auto', 'log2'],
'classifier__max_depth': [None, 10],
'classifier__bootstrap': [False, True],
'classifier__min_samples_leaf': [1, 5, 10],
'classifier__min_samples_split': [2, 5, 10],
'classifier__criterion': ['gini', 'entropy'],
}
rf_search = build_search(build_full_pipeline() , param_distributions=param_distributions, n_iter=50)
 rf_cv_results = run_search(search=rf_search)
 rf_cv_results
def score(params):
ps = {'learning_rate': params['learning_rate'],
'max_depth': params['max_depth'],
'gamma': params['gamma'],
'min_child_weight': params['min_child_weight'],
'subsample': params['subsample'],
'colsample_bytree': params['colsample_bytree'],
'verbosity': 1,
'objective': 'reg:squarederror',
'eval_metric': 'rmse',
'tree_method': 'gpu_hist',
'random_state': 27,
}
model = xgb.train(ps,d_tr, params['n_round'], [(d_val, 'eval')], early_stopping_rounds=10, verbose_eval = False)
 y_pred = model.predict(d_val)
 score = mean_squared_error(y_valid, y_pred,squared=False)
 return score<init_hyperparams>
param_distributions = {
'preprocessing__pick_columns_by_importance__n_features': [None],
'classifier': [ExtraTreesClassifier(n_estimators=500, random_state=42)],
'classifier__n_estimators': [500],
'classifier__max_features': ['auto', 'log2'],
'classifier__max_depth': [None],
'classifier__min_samples_split': [2, 5, 10],
'classifier__criterion': ['gini', 'entropy'],
}
et_search = build_search(build_full_pipeline() , param_distributions=param_distributions, n_iter=50)
 et_cv_results = run_search(search=et_search)
 et_cv_results
param_space = {'learning_rate': hp.uniform('learning_rate', 0.01, 0.3),
'n_round': scope.int(hp.quniform('n_round', 200, 3000, 100)) ,
'max_depth': scope.int(hp.quniform('max_depth', 5, 16, 1)) ,
'gamma': hp.uniform('gamma', 0, 10),
'min_child_weight': hp.uniform('min_child_weight', 0, 10),
'subsample': hp.uniform('subsample', 0.1, 1),
'colsample_bytree': hp.uniform('colsample_bytree', 0.1, 1)
 }<choose_model_class>
param_distributions = {
'preprocessing__pick_columns_by_importance__n_features': [None, 50],
'classifier': [LogisticRegression(solver='lbfgs', random_state=42)],
'classifier__multi_class': ['ovr', 'multinomial'],
'classifier__C': np.logspace(-3, 0, 4),
}
logistic_search = build_search(build_full_pipeline() , param_distributions=param_distributions, n_iter=20)
 logistic_cv_results = run_search(search=logistic_search)
 logistic_cv_results
%time
trials = Trials()
hopt = fmin(fn = score,
space = param_space,
algo = tpe.suggest,
max_evals = 1000,
trials = trials,
)<find_best_params>
param_distributions = {
'preprocessing__pick_columns_by_importance__n_features': [None, 50],
'preparation__scaler': [MinMaxScaler() ],
'classifier': [MLPClassifier(hidden_layer_sizes=(100,), random_state=42)],
'classifier__hidden_layer_sizes': [[10], [10, 10,], [10, 10, 10]],
'classifier__alpha': np.logspace(-4, -2, 3),
'classifier__solver': ['adam'],
'classifier__tol': np.logspace(-4, -2, 3),
'classifier__learning_rate_init': np.logspace(-3, -1, 3),
'classifier__activation': ['relu', 'tanh'],
}
mlp_search = build_search(build_full_pipeline() , param_distributions=param_distributions, n_iter=5)
 mlp_cv_results = run_search(search=mlp_search)
 mlp_cv_results
params_best = hopt
params_best['max_depth'] = int(hopt['max_depth'])
 n_rounds_best = int(hopt['n_round'])
 del params_best['n_round']
print(params_best)
 print(n_rounds_best )<train_model>
%time
params_best['tree_method'] = 'gpu_hist'
d = xgb.DMatrix(train[features], train[label])
 xgb_final = xgb.train(params_best,d,n_rounds_best )<predict_on_test>
param_distributions = {
'preprocessing__pick_columns_by_importance__n_features': [None],
'classifier': [ GradientBoostingClassifier(random_state=42)],
'classifier__loss': ['deviance'],
'classifier__n_estimators': [100, 300],
'classifier__max_features': ['log2', None],
'classifier__max_depth': [5, 10],
'classifier__min_samples_leaf': [1, 5, 10],
'classifier__min_samples_split': [2, 5, 10],
'classifier__learning_rate': [0.1, 0.2],
'classifier__subsample': [0.75, 0.90, 1.0]
}
gb_search = build_search(pipeline=build_full_pipeline() , param_distributions=param_distributions, n_iter=20)
 gb_cv_results = run_search(search=gb_search)
 gb_cv_results
y_pred_final = xgb_final.predict(d)
 score_final = np.sqrt(mean_squared_error(train[label], y_pred_final))
print(score_final )<load_from_csv>
param_distributions = {
'preprocessing__pick_columns_by_importance__n_features': [None],
'classifier': [ lgb.sklearn.LGBMClassifier(random_state=42, objective='multiclass')],
'classifier__boosting_type': ['gbdt', 'dart'],
'classifier__num_leaves': [20, 31, 50],
'classifier__max_depth': [-1],
'classifier__learning_rate': [0.1, 0.2],
'classifier__n_estimators': [100, 300],
'classifier__subsample': [1.0, 0.9],
'classifier__reg_alpha': [0.0, *np.logspace(-3, -2, 2)],
'classifier__reg_lambda': [0.0],
}
lgbm_search = build_search(pipeline=build_full_pipeline() , param_distributions=param_distributions, n_iter=20)
 lgbm_cv_results = run_search(search=lgbm_search)
 lgbm_cv_results
test = pd.read_csv(path + 'test.csv')
 test.set_index('id',drop=True,inplace=True)
 d_tst = xgb.DMatrix(test[features])
<train_model>
def cross_val_predictions(classifiers, X, y):

 return np.hstack([cross_val_predict(classifier, X, y, cv=5, method='predict_proba')for classifier in classifiers])
 def first_level_predictions(classifiers, X):

 return np.hstack([classifier.predict_proba(X)for classifier in classifiers])
 best_estimators = [
rf_search.best_estimator_,
et_search.best_estimator_,
gb_search.best_estimator_,
lgbm_search.best_estimator_,
xgb_search.best_estimator_
]
X_train_second_level = cross_val_predictions(classifiers=best_estimators, X=X_train, y=y_train)
 stacking_classifier = GradientBoostingClassifier(random_state=42)
 stacking_classifier.fit(X_train_second_level, y_train )
models = []
for seed in range(0,10):
params_best['seed'] = seed
xgb_final = xgb.train(params_best,d,num_boost_round = n_rounds_best)
 models.append(xgb_final)
 xgb_pred = xgb_final.predict(d_tst )<save_to_csv>
for estimator in best_estimators:
estimator.fit(X_train, y_train)
 X_val_second_level = first_level_predictions(best_estimators, X_val)
 y_val_pred = stacking_classifier.predict(X_val_second_level)
 print("Performance of stacking classifier on the hold-out set:", cohen_kappa_score(y_val, y_val_pred))
ids = test.index
output = pd.DataFrame({'id': ids,
'target': xgb_pred})
 output.to_csv('submission.csv', index=False )<define_variables>
X_train_val_second_level = cross_val_predictions(best_estimators, X=X_train_val, y=y_train_val)
 stacking_classifier.fit(X_train_val_second_level, y_train_val)
 for estimator in best_estimators:
estimator.fit(X_train_val, y_train_val)
 X_test_second_level = first_level_predictions(best_estimators, X=X_test )
input_path = Path('/kaggle/input/tabular-playground-series-jan-2021/' )<load_from_csv>
def get_predictions(estimator, X):
predictions = estimator.predict(X)
 indices = X_test.loc[:, 'PetID']
as_dict = [{'PetID': index, 'AdoptionSpeed': prediction} for index, prediction in zip(indices, predictions)]
df = pd.DataFrame.from_dict(as_dict)
 df = df.reindex(['PetID', 'AdoptionSpeed'], axis=1)
 return df
predictions = get_predictions(stacking_classifier, X=X_test_second_level )
<load_from_csv><EOS> 
def write_submission(predictions):
submission_folder = '.'
dest_file = os.path.join(submission_folder, 'submission.csv')
 predictions.to_csv(dest_file, index=False)
 print("Wrote to {}".format(dest_file))
write_submission(predictions )