Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<create_dataframe>
%matplotlib inline
plt.rc('figure', figsize=(20.0, 10.0))
GPU_AVAILABLE = tf.test.is_gpu_available()
QUADRATIC_WEIGHT_SCORER = make_scorer(cohen_kappa_score, weights='quadratic')
 print("GPU available:", GPU_AVAILABLE )
r2 = pd.DataFrame()
experiments = list()
for i in range(1, 10):
simple_blending_features_predictions, result2 = simple_blending_features(classifiers,
LogisticRegression(C=5, random_state=RANDOM_STATE),
X_train, X_test, y_train, x_tst,
part1_ratio=0.9,
random_state=i)
 r2[i] = result2
print(cohen_kappa_score(y_test, simple_blending_features_predictions, weights='quadratic'))
experiments.append(cohen_kappa_score(y_test, simple_blending_features_predictions, weights='quadratic'))
print('mean kappa: {0}
std: {1}'.format(round(np.mean(experiments), 4), round(np.std(experiments), 5)) )<data_type_conversions>
def read_sentiment_if_exists(directory: str, pet_id: str):
path = os.path.join(directory, "{}.json".format(pet_id))
if not os.path.exists(path):
return None
with open(path, 'r')as f:
return json.load(f)
 def parse_sentiment(sentiment):
documentSentiment = sentiment['documentSentiment']
return { 'SentimentMagnitude': documentSentiment['magnitude'],
'SentimentScore': documentSentiment['score'],
'SentimentLanguage': sentiment['language'] }
def read_csv_to_pandas(is_train=True):
path = os.path.join(INPUT_DIR, 'train', 'train.csv')if is_train else os.path.join(INPUT_DIR, 'test', 'test.csv')
 df = pd.read_csv(path)
 def add_sentiment_if_exists(row):
pet_id = row.get('PetID')
 directory = os.path.join(INPUT_DIR, 'train_sentiment')if is_train else os.path.join(INPUT_DIR, 'test_sentiment')
 sentiment_or_none = read_sentiment_if_exists(directory, pet_id)
 if sentiment_or_none is None:
return row.append(pd.Series({ 'SentimentMagnitude': np.nan, 'SentimentScore': np.nan, 'SentimentLanguage': "unknown" }))
sentiment_json = sentiment_or_none
sentiment = parse_sentiment(sentiment_json)
 sentiment_series = pd.Series(sentiment)
 return row.append(sentiment_series)
 df = df.apply(add_sentiment_if_exists, axis=1, result_type='expand')
 df.Name = df.Name.fillna(value="Unknown")
 return df.astype({
'Type': 'category',
'Name': 'category',
'Breed1': 'category',
'Breed2': 'category',
'Gender': 'category',
'Color1': 'category',
'Color2': 'category',
'Color3': 'category',
'MaturitySize': 'category',
'FurLength': 'category',
'Vaccinated': 'category',
'Dewormed': 'category',
'Sterilized': 'category',
'Health': 'category',
'State': 'category',
'RescuerID': 'category',
'SentimentLanguage': 'category'
})
 train_df = read_csv_to_pandas(is_train=True)
 X_test = read_csv_to_pandas(is_train=False)
 train_df.info()
r2['avg'] = r2.mean(axis=1 ).round(0 ).astype(int)
 r2.avg.head()<save_to_csv>
def to_features_and_labels(df):
y = df['AdoptionSpeed'].values
X = df.drop('AdoptionSpeed', axis=1)
 return X, y
X_train_val, y_train_val = to_features_and_labels(train_df)
 X_train, X_val, y_train, y_val = train_test_split(X_train_val, y_train_val, test_size=0.20, random_state=42,
stratify=y_train_val)
 print("Shape of X_train:", X_train.shape)
 print("Shape of X_val:", X_val.shape)
 print("Shape of y_train:", y_train.shape)
 print("Shape of y_val:", y_val.shape )
submission = pd.DataFrame(data={'PetID' : test_df['PetID'], 'AdoptionSpeed' : r['avg']})
 submission.to_csv('submission.csv', index=False)
<load_from_csv>
class DataFrameColumnDropper(BaseEstimator, TransformerMixin):

 def __init__(self, column_names):
self.column_names = column_names
def fit(self, X, y=None):
return self
def transform(self, X):
return X.copy().drop(self.column_names, axis=1)
 class ColumnImputer(BaseEstimator, TransformerMixin):
def __init__(self, column_name, imputer):
self.column_name = column_name
self.imputer = imputer
def fit(self, X, y=None):
values = X[[self.column_name]].values
self.imputer.fit(values)
 return self
def transform(self, X):
Y = X.copy()
Y.loc[:, self.column_name] = self.imputer.transform(Y[[self.column_name]].values)
 return Y
df_train = pd.read_csv('.. /input/train/train.csv')
 df_breed = pd.read_csv('.. /input/breed_labels.csv')
 df_color = pd.read_csv('.. /input/color_labels.csv')
 df_state = pd.read_csv('.. /input/state_labels.csv')
 df_test = pd.read_csv('.. /input/test/test.csv' )<data_type_conversions>
def build_preprocessing_pipeline() -> Pipeline:
return Pipeline([ 
('impute_sentiment_magnitude', ColumnImputer(column_name='SentimentMagnitude',
imputer=SimpleImputer(strategy='constant', fill_value=0.0))), 
('impute_sentiment_score', ColumnImputer(column_name='SentimentScore',
imputer=SimpleImputer(strategy='constant', fill_value=0.0))), 
('drop_unused_columns', DataFrameColumnDropper(
 column_names=['PetID', 'Description', 'RescuerID'])
)
 ])
 preprocessing_pipeline = build_preprocessing_pipeline()
X_train_preprocessed = preprocessing_pipeline.fit_transform(X_train, y_train)
 X_train_preprocessed.head(20 )
df_train.Name = df_train.Name.fillna('')
 df_test.Name = df_test.Name.fillna('' )<feature_engineering>
print("Number of features:", len(list(X_train_preprocessed)))
 print("")
 print("Numerical columns:", list(X_train_preprocessed.select_dtypes(include="number")))
 print("")
 print("Categorical columns:", list(X_train_preprocessed.select_dtypes(include="category")) )
df_train['Name'] = df_train['Name'].replace('No Name Yet', '')
 df_test['Name'] = df_test['Name'].replace('No Name Yet', '' )<feature_engineering>
class CatBoostPandasClassifier:

 def __init__(self, *args, **kwargs):
self.catboost_classifier = CatBoostClassifier(*args, **kwargs)
 self.columns = None
def fit(self, X, y, *args, **kwargs):
self.columns = list(X)
 cat_columns = list(X.select_dtypes(include='category'))
cat_features = [X.columns.get_loc(name)for name in cat_columns]
return self.catboost_classifier.fit(X.values, y, *args, cat_features=cat_features, **kwargs)
 def copy(self, *args, **kwargs):
returned_classifier = CatBoostPandasClassifier()
returned_classifier.catboost_classifier = self.catboost_classifier.copy()
returned_classifier.columns = self.columns
return returned_classifier
def predict(self, X, *args, **kwargs):
X_copy = X.loc[:, self.columns].copy()
return self.catboost_classifier.predict(X_copy.values, *args, **kwargs)
 def predict_proba(self, X, *args, **kwargs):
X_copy = X.loc[:, self.columns].copy()
return self.catboost_classifier.predict_proba(X_copy.values, *args, **kwargs)
 def __getattr__(self, attr):

 return getattr(self.catboost_classifier, attr)
 catboost_pandas_clf = CatBoostPandasClassifier(iterations=10,
learning_rate=0.1,
loss_function='MultiClass',
allow_writing_files=False)
 cross_val_score(catboost_pandas_clf, X_train_preprocessed, y_train, cv=5, scoring=QUADRATIC_WEIGHT_SCORER )
df_train['name_len'] = df_train.Name.str.len()
df_test['name_len'] = df_test.Name.str.len()<import_modules>
def build_search(pipeline, param_distributions, cv=5, n_iter=10):

 return RandomizedSearchCV(pipeline,
param_distributions=param_distributions,
cv=cv,
return_train_score=True,
refit='cohen_kappa_quadratic',
n_iter=n_iter,
n_jobs=None,
scoring={
'accuracy': make_scorer(accuracy_score),
'cohen_kappa_quadratic': QUADRATIC_WEIGHT_SCORER
},
verbose=1,
random_state=42)
 def pretty_cv_results(cv_results,
sort_by='rank_test_cohen_kappa_quadratic',
sort_ascending=True,
n_rows=30):

 df = pd.DataFrame(cv_results)
 cols_of_interest = [key for key in df.keys()
if key.startswith('param_')
 or key.startswith("mean_train")
 or key.startswith("std_train")
 or key.startswith("mean_test")
 or key.startswith("std_test")
 or key.startswith('mean_fit_time')
 or key.startswith('rank')]
return df.loc[:, cols_of_interest].sort_values(by=sort_by, ascending=sort_ascending ).head(n_rows)
 def run_search(search):
search.fit(X_train, y_train)
 print('Best score is:', search.best_score_)
 return pretty_cv_results(search.cv_results_ )
import json<define_variables>
task_type = "GPU" if GPU_AVAILABLE else "CPU"
def build_catboost_pipeline() :
return Pipeline([ 
('preprocessing', preprocessing_pipeline), 
('classifier', CatBoostPandasClassifier(verbose=0,
loss_function='MultiClass',
allow_writing_files=False,
task_type=task_type))
])
 param_distributions = {
'classifier__iterations': [500, 1000],
'classifier__learning_rate': scipy.stats.uniform(0.01, 0.3),
'classifier__max_depth': scipy.stats.randint(3, 10),
'classifier__one_hot_max_size': [30],
'classifier__l2_leaf_reg': scipy.stats.reciprocal(a=1e-2, b=1e1)
 }
catboost_search = build_search(build_catboost_pipeline() ,
param_distributions=param_distributions,
n_iter=50,
cv=RepeatedStratifiedKFold(n_splits=10, n_repeats=1, random_state=42))
catboost_cv_results = run_search(search=catboost_search)
 catboost_cv_results
train_sentiment_path = '.. /input/train_sentiment/'
test_sentiment_path = '.. /input/test_sentiment/'
train_meta_path = '.. /input/train_metadata/'
test_meta_path = '.. /input/test_metadata/'<feature_engineering>
best_estimator = catboost_search.best_estimator_
print(best_estimator.named_steps['classifier'].get_params() )
df_train[['desc_magnitude', 'desc_score']] = df_train['PetID'].progress_apply(lambda x: get_sentiment(x, train_sentiment_path))
df_test[['desc_magnitude', 'desc_score']] = df_test['PetID'].progress_apply(lambda x: get_sentiment(x, test_sentiment_path))<prepare_x_and_y>
all_estimators = [build_catboost_pipeline().set_params(**params)for params in catboost_search.cv_results_['params']]
scores_and_estimators = list(zip(catboost_search.cv_results_['mean_test_cohen_kappa_quadratic'], all_estimators))
scores_and_estimators.sort(key=lambda x: x[0], reverse=True)
 scores_and_estimators[:3]
target = df_train['AdoptionSpeed']
train_id = df_train['PetID']
test_id = df_test['PetID']
df_train.drop(['AdoptionSpeed', 'PetID'], axis=1, inplace=True)
 df_test.drop(['PetID'], axis=1, inplace=True )<prepare_x_and_y>
N_TOP_ESTIMATORS = 5
best_estimators = scores_and_estimators[:N_TOP_ESTIMATORS]
voting_classifier = VotingClassifier([(str(score), estimator)for score, estimator in best_estimators], voting='soft' )
%%time
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in train_id:
try:
with open('.. /input/train_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 print(nl_count)
 df_train.loc[:, 'vertex_x'] = vertex_xs
df_train.loc[:, 'vertex_y'] = vertex_ys
df_train.loc[:, 'bounding_confidence'] = bounding_confidences
df_train.loc[:, 'bounding_importance'] = bounding_importance_fracs
df_train.loc[:, 'dominant_blue'] = dominant_blues
df_train.loc[:, 'dominant_green'] = dominant_greens
df_train.loc[:, 'dominant_red'] = dominant_reds
df_train.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
df_train.loc[:, 'dominant_score'] = dominant_scores
df_train.loc[:, 'label_description'] = label_descriptions
df_train.loc[:, 'label_score'] = label_scores
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in test_id:
try:
with open('.. /input/test_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 df_test.loc[:, 'vertex_x'] = vertex_xs
df_test.loc[:, 'vertex_y'] = vertex_ys
df_test.loc[:, 'bounding_confidence'] = bounding_confidences
df_test.loc[:, 'bounding_importance'] = bounding_importance_fracs
df_test.loc[:, 'dominant_blue'] = dominant_blues
df_test.loc[:, 'dominant_green'] = dominant_greens
df_test.loc[:, 'dominant_red'] = dominant_reds
df_test.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
df_test.loc[:, 'dominant_score'] = dominant_scores
df_test.loc[:, 'label_description'] = label_descriptions
df_test.loc[:, 'label_score'] = label_scores<define_variables>
best_estimator = catboost_search.best_estimator_
y_val_pred_best_estimator = best_estimator.predict(X_val)
 voting_classifier.fit(X_train, y_train)
 y_val_pred_voting_classifier = voting_classifier.predict(X_val)
 print("Performance of best estimator on the hold-out set:",
cohen_kappa_score(y_val, y_val_pred_best_estimator, weights='quadratic'))
print("Performance of voting classifier on the hold-out set:",
cohen_kappa_score(y_val, y_val_pred_voting_classifier, weights='quadratic'))
image_meta_col = ['vertex_x', 'vertex_y', 'bounding_confidence', 'bounding_importance', 'dominant_blue', 'dominant_green', 'dominant_red', 'dominant_pixel_frac', 'dominant_score', 'label_score']<count_missing_values>
column_names = best_estimator.named_steps['classifier'].columns
feature_importances = best_estimator.named_steps['classifier'].feature_importances_
print("{} columns, {} feature importances.".format(len(column_names), len(feature_importances)))
 features_and_importances = list(zip(column_names, feature_importances))
features_and_importances.sort(key=lambda x: x[1], reverse=True)
 for name, feature_importance in features_and_importances:
print("{} -> {}".format(name, feature_importance))
df_train.isna().sum()<feature_engineering>
voting_classifier.fit(X_train_val, y_train_val )
def log_transform(feature, df_train, df_test):
min_feature = min(df_train[feature].min() , df_test[feature].min())
 df_train[feature] = np.log(df_train[feature] + 1 - min_feature)
 df_test[feature] = np.log(df_test[feature] + 1 - min_feature)
 return df_train, df_test<feature_engineering>
def get_predictions(estimator, X):
predictions = estimator.predict(X ).astype(np.int32)
 predictions = np.squeeze(predictions)
 indices = X.loc[:, 'PetID']
as_dict = [{'PetID': index, 'AdoptionSpeed': prediction} for index, prediction in zip(indices, predictions)]
df = pd.DataFrame.from_dict(as_dict)
 df = df.reindex(['PetID', 'AdoptionSpeed'], axis=1)
 return df
predictions = get_predictions(best_estimator, X=X_test )
<feature_engineering><EOS> 
def write_submission(predictions):
submission_folder = '.'
dest_file = os.path.join(submission_folder, 'submission.csv')
 predictions.to_csv(dest_file, index=False)
 print("Wrote to {}".format(dest_file))
write_submission(predictions )