Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<compute_train_metric>
%matplotlib inline
pd.options.display.max_rows = 128
pd.options.display.max_columns = 128
optR = OptimizedRounder()
coefficients_ = np.mean(results['coefficients'], axis=0)
 print(coefficients_)
 coefficients_[0] = 1.645
coefficients_[1] = 2.115
coefficients_[3] = 2.84
train_predictions = [r[0] for r in results['train']]
train_predictions = optR.predict(train_predictions, coefficients_ ).astype(int)
 Counter(train_predictions )<compute_train_metric>
plt.rcParams['figure.figsize'] =(18, 15 )
optR = OptimizedRounder()
coefficients_ = np.mean(results['coefficients'], axis=0)
 print(coefficients_)
 coefficients_[0] = 1.645
coefficients_[1] = 2.115
coefficients_[3] = 2.84
test_predictions = [r[0] for r in results['test']]
test_predictions = optR.predict(test_predictions, coefficients_ ).astype(int)
 Counter(test_predictions )<count_values>
train = pd.read_csv('.. /input/train/train.csv')
 test = pd.read_csv('.. /input/test/test.csv')
 sample_submission = pd.read_csv('.. /input/test/sample_submission.csv' )
print("True Distribution:")
 print(pd.value_counts(target, normalize=True ).sort_index())
 print("Test Predicted Distribution:")
 print(pd.value_counts(test_predictions, normalize=True ).sort_index())
 print("Train Predicted Distribution:")
 print(pd.value_counts(train_predictions, normalize=True ).sort_index() )<create_dataframe>
labels_breed = pd.read_csv('.. /input/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/color_labels.csv')
 labels_color = pd.read_csv('.. /input/state_labels.csv' )
pd.DataFrame(sk_cmatrix(target, train_predictions), index=list(range(5)) , columns=list(range(5)) )<compute_test_metric>
train_image_files = sorted(glob.glob('.. /input/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/train_sentiment/*.json'))
print('num of train images files: {}'.format(len(train_image_files)))
 print('num of train metadata files: {}'.format(len(train_metadata_files)))
 print('num of train sentiment files: {}'.format(len(train_sentiment_files)))
 test_image_files = sorted(glob.glob('.. /input/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/test_sentiment/*.json'))
print('num of test images files: {}'.format(len(test_image_files)))
 print('num of test metadata files: {}'.format(len(test_metadata_files)))
 print('num of test sentiment files: {}'.format(len(test_sentiment_files)) )
quadratic_weighted_kappa(target, train_predictions)
 rmse(target, [r[0] for r in results['train']])
 submission = pd.DataFrame({'PetID': test_id, 'AdoptionSpeed': test_predictions})
 submission.head()<save_to_csv>
test_df_ids = test[['PetID']]
print(test_df_ids.shape)
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 print(len(test_imgs_pets.unique()))
pets_with_images = len(np.intersect1d(test_imgs_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with images: {:.3f}'.format(pets_with_images / test_df_ids.shape[0]))
test_df_ids = test[['PetID']]
test_df_metadata = pd.DataFrame(test_metadata_files)
 test_df_metadata.columns = ['metadata_filename']
test_metadata_pets = test_df_metadata['metadata_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 test_df_metadata = test_df_metadata.assign(PetID=test_metadata_pets)
 print(len(test_metadata_pets.unique()))
pets_with_metadatas = len(np.intersect1d(test_metadata_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with metadata: {:.3f}'.format(pets_with_metadatas / test_df_ids.shape[0]))
test_df_ids = test[['PetID']]
test_df_sentiment = pd.DataFrame(test_sentiment_files)
 test_df_sentiment.columns = ['sentiment_filename']
test_sentiment_pets = test_df_sentiment['sentiment_filename'].apply(lambda x: x.split('/')[-1].split('.')[0])
 test_df_sentiment = test_df_sentiment.assign(PetID=test_sentiment_pets)
 print(len(test_sentiment_pets.unique()))
pets_with_sentiments = len(np.intersect1d(test_sentiment_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with sentiment: {:.3f}'.format(pets_with_sentiments / test_df_ids.shape[0]))
print('images and metadata distributions the same? {}'.format(
 np.all(test_metadata_pets == test_imgs_pets)) )
submission.to_csv('submission.csv', index=False )<save_to_csv>
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_metadata_file(self, filename):

 with open(filename, 'r')as f:
metadata_file = json.load(f)
 return metadata_file
def open_sentiment_file(self, filename):

 with open(filename, 'r')as f:
sentiment_file = json.load(f)
 return sentiment_file
def open_image_file(self, filename):

 image = np.asarray(Image.open(filename))
return image
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 if self.extract_sentiment_text:
file_sentences_text = [x['text']['content'] for x in file['sentences']]
file_sentences_text = self.sentence_sep.join(file_sentences_text)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns' ).sum()
file_sentences_sentiment = file_sentences_sentiment.add_prefix('document_' ).to_dict()
file_sentiment.update(file_sentences_sentiment)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
if self.extract_sentiment_text:
df_sentiment['text'] = file_sentences_text
df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations'][:int(len(file['labelAnnotations'])* 0.3)]
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = '.. /input/{}_sentiment/{}.json'.format(mode, pet_id)
 try:
sentiment_file = pet_parser.open_sentiment_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob('.. /input/{}_metadata/{}*.json'.format(mode, pet_id)))
 if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_metadata_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()
submission.to_csv('submission.csv', index=False )<create_dataframe>
aggregates = ['mean', 'sum', 'var']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(aggregates)
 train_sentiment_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(aggregates)
 test_sentiment_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()
%matplotlib inline
dfk = pd.DataFrame({
'Kernel ID': ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q'],
'Score': [ 0.69864 , 0.69846 , 0.69836 , 0.69824 , 0.69813, 0.69795, 0.69752, 0.69751, 0.69749, 0.69747, 0.69735, 0.69731, 0.69703, 0.69701, 0.69691, 0.69648, 0.69579],
'File Path': ['.. /input/aa69864/AA69864.csv', '.. /input/bb69846/BB69846.csv', '.. /input/cc69836/CC69836.csv', '.. /input/a69824/A69824.csv', '.. /input/c69813/C69813.csv', '.. /input/ff69795/FF69795.csv', '.. /input/ggg69752/GGG69752.csv', '.. /input/gg69751/GG69751.csv' , '.. /input/g69749/G69749.csv', '.. /input/h69747/H69747.csv', '.. /input/i69735/I69735.csv', '.. /input/j69731/J69731.csv', '.. /input/mmm69703/MMM69703.csv', '.. /input/l69701/L69701.csv', '.. /input/ooo69691/OOO69691.csv', '.. /input/ppp69648/PPP69648.csv', '.. /input/jan21-tabular-playground-4-lb-final-blend/submission.csv']
})
 dfk<categorify>
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]
def generate(main, support, coeff):
g = main.copy()
for i in main.columns[1:]:
res = []
lm, Is = [], []
lm = main[i].tolist()
ls = support[i].tolist()
for j in range(len(main)) :
res.append(( lm[j] * coeff)+(ls[j] *(1.- coeff)))
 g[i] = res
return g 
<load_from_csv>
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )
support = pd.read_csv(dfk.iloc[0, 2])
 for k in range(1, 11):
main = pd.read_csv(dfk.iloc[k, 2])
 support = generate(main, support, 0.60)
 sub1 = support<compute_test_metric>
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False)
 print('NaN structure:
{}'.format(np.sum(pd.isnull(X))))
sub8 = comparison(sub7, 8, 1.0030, 0.9980)
<save_to_csv>
column_types = X.dtypes
int_cols = column_types[column_types == 'int']
float_cols = column_types[column_types == 'float']
cat_cols = column_types[column_types == 'object']
print('\tinteger columns:
{}'.format(int_cols))
print('
\tfloat columns:
{}'.format(float_cols))
print('
\tto encode categorical columns:
{}'.format(cat_cols))
sub = sub8
sub.to_csv("submission.csv", index=False)
 sub1.to_csv("submission1.csv", index=False)
 sub2.to_csv("submission2.csv", index=False)
 sub3.to_csv("submission3.csv", index=False)
 sub4.to_csv("submission4.csv", index=False)
 sub5.to_csv("submission5.csv", index=False)
 sub6.to_csv("submission6.csv", index=False)
 sub7.to_csv("submission7.csv", index=False)
 !ls<import_modules>
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID']
import pandas as pd
import numpy as np
import lightgbm as lgbm
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error 
<load_from_csv>
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID' )
train = pd.read_csv(".. /input/tabular-playground-series-jan-2021/train.csv")
 test = pd.read_csv(".. /input/tabular-playground-series-jan-2021/test.csv" )<define_variables>
for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]
cont_features = [col for col in train.columns if col.startswith("cont")]
len(cont_features )<split>
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('<MISSING>' )
y = train["target"]
kf = KFold(n_splits=5, shuffle=True, random_state=1)
 oof = np.zeros(len(train))
score_list = []
fold = 1
test_preds = []
seed_list = [None,2,3]
for train_index, test_index in kf.split(train):
X_train, X_val = train.iloc[train_index], train.iloc[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
X_train = X_train.abs()
y_pred_list = []
for seed in seed_list:
dtrain = lgbm.Dataset(X_train[cont_features], y_train)
 dvalid = lgbm.Dataset(X_val[cont_features], y_val)
 print(seed)
 params = {"objective": "regression",
"metric": "rmse",
"verbosity": -1,
"boosting_type": "gbdt",
"feature_fraction":0.5,
"num_leaves": 250,
"lambda_l1":7,
"lambda_l2":2,
"learning_rate":0.01,
'min_child_samples': 35,
"bagging_fraction":0.75,
"bagging_freq":1,
}
params["seed"] = seed
model = lgbm.train(params,
dtrain,
valid_sets=[dtrain, dvalid],
verbose_eval=100,
num_boost_round=100000,
early_stopping_rounds=100 
)
 dtrain = lgbm.Dataset(X_train[cont_features], y_train)
 dvalid = lgbm.Dataset(X_val[cont_features], y_val)
 params = {"objective": "regression",
"metric": "rmse",
"verbosity": -1,
"boosting_type": "gbdt",
"feature_fraction":0.5,
"num_leaves": 350,
"lambda_l1":7,
"lambda_l2":1,
"learning_rate":0.003,
'min_child_samples': 35,
"bagging_fraction":0.8,
"bagging_freq":1,
}
params["seed"] = seed
model = lgbm.train(params,
dtrain,
valid_sets=[dtrain, dvalid],
verbose_eval=100,
num_boost_round=100000,
early_stopping_rounds=200,
init_model = model 
)
 y_pred_list.append(model.predict(X_val[cont_features]))
print(np.sqrt(mean_squared_error(y_val, np.mean(y_pred_list,axis=0))))
test_preds.append(model.predict(test[cont_features]))
oof[test_index] = np.mean(y_pred_list,axis=0)
 score = np.sqrt(mean_squared_error(y_val, oof[test_index]))
score_list.append(score)
 print(f"RMSE Fold-{fold} : {score}")
 fold+=1
np.mean(score_list )<compute_test_metric>
n_components = 5
text_features = []
for i in X_text.columns:
print('generating features from: {}'.format(i))
svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 nmf_ = NMF(
 n_components=n_components, random_state=1337)
 tfidf_col = TfidfVectorizer().fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('SVD_{}_'.format(i))
nmf_col = nmf_.fit_transform(tfidf_col)
 nmf_col = pd.DataFrame(nmf_col)
 nmf_col = nmf_col.add_prefix('NMF_{}_'.format(i))
text_features.append(svd_col)
 text_features.append(nmf_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )
print(score_list)
 np.mean(score_list )<feature_engineering>
np.sum(pd.isnull(X_train))
train["1_preds"] = oof
test["1_preds"] = np.mean(test_preds,axis=0 )<feature_engineering>
np.sum(pd.isnull(X_test))
threshold = 8
train["target_class"] = train["target"].apply(lambda x: 1 if x > threshold else 0)
 train.groupby("target_class")["target"].mean()<split>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator)
 class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))
y = train["target_class"]
kf = KFold(n_splits=5, shuffle=True, random_state=1)
 oof = np.zeros(len(train))
score_list = []
fold = 1
test_preds = []
for train_index, test_index in kf.split(train):
X_train, X_val = train.iloc[train_index], train.iloc[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
dtrain_0 = lgbm.Dataset(X_train[cont_features], y_train)
 dvalid = lgbm.Dataset(X_val[cont_features], y_val)
 params = {"objective": "binary",
"metric": "auc",
"verbosity": -1,
"boosting_type": "gbdt",
"feature_fraction":0.5,
"num_leaves": 200,
"lambda_l1":7,
"lambda_l2":0,
"learning_rate":0.01,
'min_child_samples': 0,
"bagging_fraction":0.75,
"bagging_freq":1,
}
model_0 = lgbm.train(params,
dtrain_0,
valid_sets=[dtrain_0, dvalid],
verbose_eval=100,
num_boost_round=100000,
early_stopping_rounds=100 
)
 y_pred_0 = model_0.predict(X_val[cont_features])
 score = roc_auc_score(y_val,y_pred_0)
 print(f"AUC Fold-{fold} : {score}")
 oof[test_index] = y_pred_0
score_list.append(score)
 test_preds.append(model_0.predict(test[cont_features]))
fold+=1
np.mean(score_list )<compute_test_metric>
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 70,
'max_depth': 9,
'learning_rate': 0.01,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.02,
'min_child_samples': 150,
'min_child_weight': 0.02,
'lambda_l2': 0.0475,
'verbosity': -1,
'data_random_seed': 17}
early_stop = 500
verbose_eval = 100
num_rounds = 90000
n_splits = 10
train["class_preds"] = oof
test["class_preds"] = np.mean(test_preds,axis=0)
 roc_auc_score(train["target_class"],train["class_preds"] )<split>
kfold = StratifiedKFold(n_splits=n_splits, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_index, valid_index in kfold.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_index, :]
X_val = X_train.iloc[valid_index, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 print('
y_tr distribution: {}'.format(Counter(y_tr)))
 d_train = lgb.Dataset(X_tr, label=y_tr)
 d_valid = lgb.Dataset(X_val, label=y_val)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 val_pred = model.predict(X_val, num_iteration=model.best_iteration)
 test_pred = model.predict(X_test, num_iteration=model.best_iteration)
 oof_train[valid_index] = val_pred
oof_test[:, i] = test_pred
i += 1
y = train["target"]
kf = KFold(n_splits=5, shuffle=True, random_state=1)
 oof_2 = np.zeros(len(train))
score_list = []
fold = 1
test_preds_2 = []
for train_index, test_index in kf.split(train):
X_train, X_val = train.iloc[train_index], train.iloc[test_index]
y_train, y_val = y.iloc[train_index], y.iloc[test_index]
X_train_0 = X_train[X_train.target_class==0]
y_train_0 = y_train[X_train.target_class==0]
X_train_1 = X_train[X_train.target_class==1]
y_train_1 = y_train[X_train.target_class==1]
dtrain_0 = lgbm.Dataset(X_train_0[cont_features], y_train_0)
 dvalid = lgbm.Dataset(X_val[cont_features], y_val)
 params = {"objective": "regression",
"metric": "rmse",
"verbosity": -1,
"boosting_type": "gbdt",
"feature_fraction":0.5,
"num_leaves": 250,
"lambda_l1":4,
"lambda_l2":2,
"learning_rate":0.01,
'min_child_samples': 35,
"bagging_fraction":0.75,
"bagging_freq":1,
}
model_0 = lgbm.train(params,
dtrain_0,
valid_sets=[dtrain_0, dvalid],
verbose_eval=100,
num_boost_round=100000,
early_stopping_rounds=100 
)
 y_pred_0 = model_0.predict(X_val[cont_features])
 score = np.sqrt(mean_squared_error(y_val, y_pred_0))
print(f"RMSE Fold-{fold} : {score}")
 dtrain_1 = lgbm.Dataset(X_train_1[cont_features], y_train_1)
 dvalid = lgbm.Dataset(X_val[cont_features], y_val)
 params = {"objective": "regression",
"metric": "rmse",
"verbosity": -1,
"boosting_type": "gbdt",
"feature_fraction":0.5,
"num_leaves": 250,
"lambda_l1":4,
"lambda_l2":2,
"learning_rate":0.01,
'min_child_samples': 35,
"bagging_fraction":0.75,
"bagging_freq":1,
}
model_1 = lgbm.train(params,
dtrain_1,
valid_sets=[dtrain_1, dvalid],
verbose_eval=100,
num_boost_round=100000,
early_stopping_rounds=100 
)
 y_pred_1 = model_1.predict(X_val[cont_features])
 score = np.sqrt(mean_squared_error(y_val, y_pred_1))
print(f"RMSE Fold-{fold} : {score}")
 oof_2[test_index] = y_pred_0*(1-X_val["class_preds"])+ y_pred_1*(X_val["class_preds"])
 score = np.sqrt(mean_squared_error(y_val,oof_2[test_index]))
print(f"RMSE Fold-Final-{fold} : {score}")
 score_list.append(score)
 test_preds_2.append(model_0.predict(test[cont_features])*(1-test["class_preds"])+ model_1.predict(test[cont_features])*(test["class_preds"]))
fold+=1
np.mean(score_list )<compute_test_metric>
importance_type= "split"
idx_sort = np.argsort(model.feature_importance(importance_type=importance_type)) [::-1]
names_sorted = np.array(model.feature_name())[idx_sort]
imports_sorted = model.feature_importance(importance_type=importance_type)[idx_sort]
for n, im in zip(names_sorted, imports_sorted):
print(n, im )
print(np.mean(score_list))
score_list<compute_test_metric>
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(oof_train, coefficients)
 print("
Valid Counts = ", Counter(X_train['AdoptionSpeed'].values))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, pred_test_y_k)
 print("QWK = ", qwk )
train["2_preds"] = oof_2
test["2_preds"] = np.mean(test_preds_2,axis=0)
 np.sqrt(mean_squared_error(train["target"],train["2_preds"]))<import_modules>
coefficients_ = coefficients.copy()
coefficients_[0] = 1.645
coefficients_[1] = 2.115
coefficients_[3] = 2.84
train_predictions = optR.predict(oof_train, coefficients_ ).astype(int)
 print('train pred distribution: {}'.format(Counter(train_predictions)))
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_)
 print('test pred distribution: {}'.format(Counter(test_predictions)) )
from tensorflow.keras.layers import Input,Dense,Dropout
from tensorflow.keras import Model
from tensorflow.keras.regularizers import l2
import tensorflow as tf<choose_model_class>
print("True Distribution:")
 print(pd.value_counts(X_train['AdoptionSpeed'], normalize=True ).sort_index())
 print("
Train Predicted Distribution:")
 print(pd.value_counts(train_predictions, normalize=True ).sort_index())
 print("
Test Predicted Distribution:")
 print(pd.value_counts(test_predictions, normalize=True ).sort_index() )
<categorify><EOS> 
submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions.astype(np.int32)})
 submission.head()
submission.to_csv('submission.csv', index=False )