Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<define_variables>
import numpy as np
import pandas as pd
import os
import xgboost as xgb
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.preprocessing import LabelEncoder
from scipy import sparse
from sklearn.decomposition import TruncatedSVD
from sklearn.model_selection import train_test_split, KFold, StratifiedKFold
import scipy as sp
from sklearn import linear_model
from functools import partial
from sklearn import metrics
from collections import Counter
import json
import lightgbm as lgb
train_df_ids = train[['PetID']]
test_df_ids = test[['PetID']]
train_df_imgs = pd.DataFrame(train_image_files)
 train_df_imgs.columns = ['image_filename']
train_imgs_pets = train_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 def getSize(filename):
st = os.stat(filename)
 return st.st_size
def getDimensions(filename):
img_size = Image.open(filename ).size
return img_size
train_df_imgs['image_size'] = train_df_imgs['image_filename'].apply(getSize)
 train_df_imgs['temp_size'] = train_df_imgs['image_filename'].apply(getDimensions)
 train_df_imgs['width'] = train_df_imgs['temp_size'].apply(lambda x : x[0])
 train_df_imgs['height'] = train_df_imgs['temp_size'].apply(lambda x : x[1])
 train_df_imgs = train_df_imgs.drop(['temp_size'], axis=1)
 test_df_imgs['image_size'] = test_df_imgs['image_filename'].apply(getSize)
 test_df_imgs['temp_size'] = test_df_imgs['image_filename'].apply(getDimensions)
 test_df_imgs['width'] = test_df_imgs['temp_size'].apply(lambda x : x[0])
 test_df_imgs['height'] = test_df_imgs['temp_size'].apply(lambda x : x[1])
 test_df_imgs = test_df_imgs.drop(['temp_size'], axis=1)
 aggs = {
'image_size': ['sum', 'mean', 'var'],
'width': ['sum', 'mean', 'var'],
'height': ['sum', 'mean', 'var'],
}
agg_train_imgs = train_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()
agg_test_imgs = test_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index()
agg_imgs = pd.concat([agg_train_imgs, agg_test_imgs], axis=0 ).reset_index(drop=True )<merge>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )
X_temp = X_temp.merge(agg_imgs, how='left', on='PetID' )<drop_column>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
X_temp = X_temp.drop(to_drop_columns, axis=1 )<define_variables>
train = pd.read_csv('.. /input/train/train.csv')
 test = pd.read_csv('.. /input/test/test.csv' )
X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1 )<count_missing_values>
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in train.PetID.values:
try:
with open('.. /input/train_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1 )
X_train_non_null.isnull().any().any() , X_test_non_null.isnull().any().any()<import_modules>
train['doc_sent_mag'] = doc_sent_mag
train['doc_sent_score'] = doc_sent_score
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<train_on_grid>
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in test.PetID.values:
try:
with open('.. /input/test_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1 )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']<compute_train_metric>
test['doc_sent_mag'] = doc_sent_mag
test['doc_sent_score'] = doc_sent_score
def softmax(score):
score = np.asarray(score, dtype=float)
 score = np.exp(score-np.max(score))
score /= np.sum(score, axis=1)[:,np.newaxis]
return score
def softkappaobj(preds, dtrain):
labels = dtrain.get_label() + 1
labels = np.asarray(labels, dtype=int)
 preds = softmax(preds)
 M = preds.shape[0]
N = preds.shape[1]
O = 0.0
for j in range(N):
wj =(labels -(j+1.))**2
O += np.sum(wj * preds[:,j])
 hist_label = np.bincount(labels)[1:]
hist_pred = np.sum(preds, axis=0)
 E = 0.0
for i in range(N):
for j in range(N):
E += pow(i - j, 2.0)* hist_label[i] * hist_pred[j]
grad = np.zeros(( M, N))
hess = np.zeros(( M, N))
for n in range(N):
dO = np.zeros(( M))
for j in range(N):
indicator = float(n == j)
 dO +=(( labels -(j+1.))**2)* preds[:,n] *(indicator - preds[:,j])
 dE = np.zeros(( M))
for k in range(N):
for l in range(N):
indicator = float(n == k)
 dE += pow(k-l, 2.0)* hist_label[l] * preds[:,n] *(indicator - preds[:,k])
 grad[:,n] = -M *(dO * E - O * dE)/(E**2)
 d2O = np.zeros(( M))
for j in range(N):
indicator = float(n == j)
 d2O +=(( labels -(j+1.))**2)* preds[:,n] *(1 - 2.*preds[:,n])*(indicator - preds[:,j])
 d2E = np.zeros(( M))
for k in range(N):
for l in range(N):
indicator = float(n == k)
 d2E += pow(k-l, 2.0)* hist_label[l] * preds[:,n] *(1 - 2.*preds[:,n])*(indicator - preds[:,k])
 hess[:,n] = -M *(( d2O * E - O * d2E)*(E**2)-(dO * E - O * dE)* 2.* E * dE)/(E**4)
 grad *= -1.
 hess *= -1.
 scale = 0.000125 / np.mean(abs(hess))
hess *= scale
hess = np.abs(hess)
 grad.shape =(M*N)
 hess.shape =(M*N)
 return grad, hess
def evalerror(preds, dtrain):
labels = dtrain.get_label() +1
preds = softmax(preds)
 pred_labels = np.argmax(preds, axis=1)+1
kappa = cohen_kappa_score(labels, pred_labels, weights='quadratic')
 return 'kappa', kappa<init_hyperparams>
lbl_enc = LabelEncoder()
lbl_enc.fit(train.RescuerID.values.tolist() + test.RescuerID.values.tolist())
 train.RescuerID = lbl_enc.transform(train.RescuerID.values)
 test.RescuerID = lbl_enc.transform(test.RescuerID.values )
xgb_params = {
'objective': 'reg:linear',
'num_class': 5,
'eval_metric': 'mlogloss',
'booster' : 'gbtree',
'lambda' : 0.00005,
'alpha' : 0.000001,
'seed': 48,
'eta': 0.0123,
'subsample': 0.8,
'device': 'gpu',
'silent': 1,
}<split>
train_desc = train.Description.fillna("none" ).values
test_desc = test.Description.fillna("none" ).values
tfv = TfidfVectorizer(min_df=3, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1,
stop_words = 'english')
 tfv.fit(list(train_desc)+ list(test_desc))
X = tfv.transform(train_desc)
 X_test = tfv.transform(test_desc)
 svd = TruncatedSVD(n_components=180)
 svd.fit(X)
 X = svd.transform(X)
 X_test = svd.transform(X_test )
def run_xgb(params, X_train, X_test):
n_splits = 5
verbose_eval = 2000
num_rounds = 100000
early_stop = 2000
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=48)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,early_stopping_rounds=early_stop,verbose_eval=verbose_eval,params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test<train_model>
y = train.AdoptionSpeed
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null )<compute_train_metric>
y.value_counts()
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 print(coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk)
 qwk = cohen_kappa_score(X_train['AdoptionSpeed'].values, valid_pred, weights='quadratic')
 print("QWK1 = ", qwk )<predict_on_test>
train = np.hstack(( train.drop(['Name', 'Description', 'PetID', 'AdoptionSpeed'], axis=1 ).values, X))
test = np.hstack(( test.drop(['Name', 'Description', 'PetID'], axis=1 ).values, X_test))
coefficients_ = coefficients.copy()
coefficients_[0] = 0.45897549
coefficients_[1] = 1.92101968
coefficients_[2] = 2.35620132
coefficients_[3] = 3.26394093
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}' )<count_values>
train_predictions = np.zeros(( train.shape[0], 1))
test_predictions = np.zeros(( test.shape[0], 1))
zero_test_predictions = np.zeros(( test.shape[0], 1))
FOLDS = 3
print("stratified k-folds")
 skf = StratifiedKFold(n_splits=FOLDS, random_state=42, shuffle=True)
 skf.get_n_splits(train, y)
 cv_scores = []
fold = 1
coefficients = np.zeros(( FOLDS, 4))
for train_idx, valid_idx in skf.split(train, y):
xtrain, xvalid = train[train_idx], train[valid_idx]
xtrain_text, xvalid_text = X[train_idx], X[valid_idx]
ytrain, yvalid = y.iloc[train_idx], y.iloc[valid_idx]
w = y.value_counts()
weights = {i : np.sum(w)/ w[i] for i in w.index}
print(weights)
 lgb_params = {
'boosting_type': 'gbdt',
'objective': 'regression',
'learning_rate': 0.005,
'subsample':.8,
'colsample_bytree': 0.8,
'min_split_gain': 0.006,
'min_child_samples': 150,
'min_child_weight': 0.1,
'max_depth': 17,
'n_estimators': 10000,
'num_leaves': 80,
'silent': -1,
'verbose': -1,
'max_depth': 11,
'random_state': 2018
}
model = lgb.LGBMRegressor(**lgb_params)
 model.fit(
 xtrain, ytrain,
eval_set=[(xvalid, yvalid)],
eval_metric='rmse',
verbose=100,
early_stopping_rounds=100 
)
 valid_preds = model.predict(xvalid, num_iteration=model.best_iteration_)
 optR = OptimizedRounder()
optR.fit(valid_preds, yvalid.values)
 coefficients[fold-1,:] = optR.coefficients()
valid_p = optR.predict(valid_preds, coefficients[fold-1,:])
 print("Valid Counts = ", Counter(yvalid.values))
print("Predicted Counts = ", Counter(valid_p))
test_preds = model.predict(test, num_iteration=model.best_iteration_)
 scr = quadratic_weighted_kappa(yvalid.values, valid_p)
 cv_scores.append(scr)
 print("Fold = {}.QWK = {}.Coef = {}".format(fold, scr, coefficients[fold-1,:]))
print("
")
 train_predictions[valid_idx] = valid_preds.reshape(-1, 1)
 test_predictions += test_preds.reshape(-1, 1)
 fold += 1
test_predictions = test_predictions * 1./FOLDS
print("Mean Score: {}.Std Dev: {}.Mean Coeff: {}".format(np.mean(cv_scores), np.std(cv_scores), np.mean(coefficients, axis=0)) )
Counter(train_predictions )<count_values>
optR = OptimizedRounder()
train_predictions = np.array([item for sublist in train_predictions for item in sublist])
 optR.fit(train_predictions, y)
 coefficients = optR.coefficients()
print(quadratic_weighted_kappa(y, optR.predict(train_predictions, coefficients)))
 predictions = optR.predict(test_predictions, coefficients ).astype(int)
 predictions = [item for sublist in predictions for item in sublist]
Counter(test_predictions )<save_to_csv>
sample = pd.read_csv('.. /input/test/sample_submission.csv' )
submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions})
 submission.to_csv('submission.csv', index=False)
 submission.head()<set_options>
sample.AdoptionSpeed = predictions
%matplotlib inline
np.random.seed(seed=1337)
 warnings.filterwarnings('ignore')
 split_char = '/'<load_from_csv>
sample.to_csv('submission.csv', index=False )
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv' )<import_modules>
sample.AdoptionSpeed.value_counts()