Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<categorify>
%matplotlib inline
np.random.seed(seed=1337)
 warnings.filterwarnings('ignore')
 split_char = '/'
def decode_mask(mask, shape=(768, 768)) :
pixels = mask.T.flatten()
pixels = np.concatenate([[0], pixels, [0]])
 runs = np.where(pixels[1:] != pixels[:-1])[0] + 1
runs[1::2] -= runs[::2]
return ' '.join(str(x)for x in runs )<define_variables>
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv' )
ship_list_dict = []
for name in test_names_nothing:
ship_list_dict.append({'ImageId':name,'EncodedPixels':np.nan} )<categorify>
import cv2
import os
from keras.applications.densenet import preprocess_input, DenseNet121
def enc_test(yp, y, name):
masks = split_mask(yp)
 if(len(masks)== 0):
ship_list_dict.append({'ImageId':name,'EncodedPixels':np.nan})
 for mask in masks:
ship_list_dict.append({'ImageId':name,'EncodedPixels':decode_mask(mask)} )<save_to_csv>
img_size = 256
batch_size = 256
model_pred_aug(learn, md.test_dl, enc_test, trms_dihedral)
 pred_df = pd.DataFrame(ship_list_dict)
 pred_df.to_csv('submission.csv', index=False )<define_variables>
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights=".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )
ship_dir = '.. /input'
train_image_dir = os.path.join(ship_dir, 'train')
 test_image_dir = os.path.join(ship_dir, 'test' )<prepare_output>
pet_ids = train['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
test_files =[f for f in os.listdir(test_image_dir)]
submission_df = pd.DataFrame({'ImageId':test_files})
 submission_df['EncodedPixels']=None
print(f"There are {len(test_files)} images in the test dataset")
 submission_df.head()<save_to_csv>
train_feats = pd.DataFrame.from_dict(features, orient='index')
 train_feats.columns = [f'pic_{i}' for i in range(train_feats.shape[1])]
submission_df.to_csv('submission.csv',index=False )<import_modules>
pet_ids = test['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/test_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
np.random.seed(369 )<compute_test_metric>
test_feats = pd.DataFrame.from_dict(features, orient='index')
 test_feats.columns = [f'pic_{i}' for i in range(test_feats.shape[1])]
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<compute_test_metric>
train_feats = train_feats.reset_index()
train_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 test_feats = test_feats.reset_index()
test_feats.rename({'index': 'PetID'}, axis='columns', inplace=True )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<compute_test_metric>
all_ids = pd.concat([train, test], axis=0, ignore_index=True, sort=False)[['PetID']]
all_ids.shape
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))<load_from_csv>
n_components = 32
svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 features_df = pd.concat([train_feats, test_feats], axis=0)
 features = features_df[[f'pic_{i}' for i in range(256)]].values
svd_col = svd_.fit_transform(features)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('IMG_SVD_')
 img_features = pd.concat([all_ids, svd_col], axis=1 )
print('Train')
 train = pd.read_csv(".. /input/train/train.csv")
 print(train.shape)
 print('Test')
 test = pd.read_csv(".. /input/test/test.csv")
 print(test.shape)
 print('Breeds')
 breeds = pd.read_csv(".. /input/breed_labels.csv")
 print(breeds.shape)
 print('Colors')
 colors = pd.read_csv(".. /input/color_labels.csv")
 print(colors.shape)
 print('States')
 states = pd.read_csv(".. /input/state_labels.csv")
 print(states.shape )<prepare_x_and_y>
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )
target = train['AdoptionSpeed']
train_id = train['PetID']
test_id = test['PetID']
train.drop(['AdoptionSpeed', 'PetID'], axis=1, inplace=True)
 test.drop(['PetID'], axis=1, inplace=True )<feature_engineering>
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print(f'num of train images files: {len(train_image_files)}')
 print(f'num of train metadata files: {len(train_metadata_files)}')
 print(f'num of train sentiment files: {len(train_sentiment_files)}')
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print(f'num of test images files: {len(test_image_files)}')
 print(f'num of test metadata files: {len(test_metadata_files)}')
 print(f'num of test sentiment files: {len(test_sentiment_files)}' )
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in train_id:
try:
with open('.. /input/train_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 train.loc[:, 'doc_sent_mag'] = doc_sent_mag
train.loc[:, 'doc_sent_score'] = doc_sent_score
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in test_id:
try:
with open('.. /input/test_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 test.loc[:, 'doc_sent_mag'] = doc_sent_mag
test.loc[:, 'doc_sent_score'] = doc_sent_score 
<normalization>
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_json_file(self, filename):
with open(filename, 'r', encoding='utf-8')as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns')
 file_sentences_sentiment_df = pd.DataFrame(
 {
'magnitude_sum': file_sentences_sentiment['magnitude'].sum(axis=0),
'score_sum': file_sentences_sentiment['score'].sum(axis=0),
'magnitude_mean': file_sentences_sentiment['magnitude'].mean(axis=0),
'score_mean': file_sentences_sentiment['score'].mean(axis=0),
'magnitude_var': file_sentences_sentiment['magnitude'].var(axis=0),
'score_var': file_sentences_sentiment['score'].var(axis=0),
}, index=[0] 
)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
df_sentiment = pd.concat([df_sentiment, file_sentences_sentiment_df], axis=1)
 df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = f'.. /input/petfinder-adoption-prediction/{mode}_sentiment/{pet_id}.json'
try:
sentiment_file = pet_parser.open_json_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob(f'.. /input/petfinder-adoption-prediction/{mode}_metadata/{pet_id}*.json'))
if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_json_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()
train_desc = train.Description.fillna("none" ).values
test_desc = test.Description.fillna("none" ).values
tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1, 
)
 tfv.fit(list(train_desc))
X = tfv.transform(train_desc)
 X_test = tfv.transform(test_desc)
 svd = TruncatedSVD(n_components=200)
 svd.fit(X)
 print(svd.explained_variance_ratio_.sum())
 print(svd.explained_variance_ratio_)
 X = svd.transform(X)
 X = pd.DataFrame(X, columns=['svd_{}'.format(i)for i in range(120)])
 train = pd.concat(( train, X), axis=1)
 X_test = svd.transform(X_test)
 X_test = pd.DataFrame(X_test, columns=['svd_{}'.format(i)for i in range(120)])
 test = pd.concat(( test, X_test), axis=1 )<normalization>
aggregates = ['sum', 'mean', 'var']
sent_agg = ['sum']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 train_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 test_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()
train_desc = train.Description.fillna("none" ).values
test_desc = test.Description.fillna("none" ).values
tfv = TfidfVectorizer(min_df=3, max_features=10000,
strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1,
stop_words = 'english')
 tfv.fit(list(train_desc))
X = tfv.transform(train_desc)
 X_test = tfv.transform(test_desc)
 svd = TruncatedSVD(n_components=120)
 svd.fit(X)
 print(svd.explained_variance_ratio_.sum())
 print(svd.explained_variance_ratio_)
 X = svd.transform(X)
 X = pd.DataFrame(X, columns=['svd_{}'.format(i)for i in range(120)])
 train = pd.concat(( train, X), axis=1)
 X_test = svd.transform(X_test)
 X_test = pd.DataFrame(X_test, columns=['svd_{}'.format(i)for i in range(120)])
 test = pd.concat(( test, X_test), axis=1)
<prepare_x_and_y>
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in train_id:
try:
with open('.. /input/train_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 print(nl_count)
 train.loc[:, 'vertex_x'] = vertex_xs
train.loc[:, 'vertex_y'] = vertex_ys
train.loc[:, 'bounding_confidence'] = bounding_confidences
train.loc[:, 'bounding_importance'] = bounding_importance_fracs
train.loc[:, 'dominant_blue'] = dominant_blues
train.loc[:, 'dominant_green'] = dominant_greens
train.loc[:, 'dominant_red'] = dominant_reds
train.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
train.loc[:, 'dominant_score'] = dominant_scores
train.loc[:, 'label_description'] = label_descriptions
train.loc[:, 'label_score'] = label_scores
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in test_id:
try:
with open('.. /input/test_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 test.loc[:, 'vertex_x'] = vertex_xs
test.loc[:, 'vertex_y'] = vertex_ys
test.loc[:, 'bounding_confidence'] = bounding_confidences
test.loc[:, 'bounding_importance'] = bounding_importance_fracs
test.loc[:, 'dominant_blue'] = dominant_blues
test.loc[:, 'dominant_green'] = dominant_greens
test.loc[:, 'dominant_red'] = dominant_reds
test.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
test.loc[:, 'dominant_score'] = dominant_scores
test.loc[:, 'label_description'] = label_descriptions
test.loc[:, 'label_score'] = label_scores<drop_column>
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )
train.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True)
 test.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True )<data_type_conversions>
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False )
numeric_cols = ['Age', 'Quantity', 'Fee', 'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'doc_sent_mag', 'doc_sent_score', 'dominant_score', 'dominant_pixel_frac', 'dominant_red', 'dominant_green', 'dominant_blue', 'bounding_importance', 'bounding_confidence', 'vertex_x', 'vertex_y', 'label_score'] + ['svd_{}'.format(i)for i in range(120)]
cat_cols = list(set(train.columns)- set(numeric_cols))
train.loc[:, cat_cols] = train[cat_cols].astype('category')
 test.loc[:, cat_cols] = test[cat_cols].astype('category')
 print(train.shape)
 print(test.shape)
 train.head()<find_best_model_class>
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID']
def run_cv_model(train, test, target, model_fn, params={}, eval_fn=None, label='model'):
kf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)
 fold_splits = kf.split(train, target)
 cv_scores = []
qwk_scores = []
pred_full_test = 0
pred_train = np.zeros(( train.shape[0], 5))
all_coefficients = np.zeros(( 5, 4))
feature_importance_df = pd.DataFrame()
i = 1
for dev_index, val_index in fold_splits:
print('Started ' + label + ' fold ' + str(i)+ '/5')
 if isinstance(train, pd.DataFrame):
dev_X, val_X = train.iloc[dev_index], train.iloc[val_index]
dev_y, val_y = target[dev_index], target[val_index]
else:
dev_X, val_X = train[dev_index], train[val_index]
dev_y, val_y = target[dev_index], target[val_index]
params2 = params.copy()
pred_val_y, pred_test_y, importances, coefficients, qwk = model_fn(dev_X, dev_y, val_X, val_y, test, params2)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index] = pred_val_y
all_coefficients[i-1, :] = coefficients
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 qwk_scores.append(qwk)
 print(label + ' cv score {}: RMSE {} QWK {}'.format(i, cv_score, qwk))
fold_importance_df = pd.DataFrame()
fold_importance_df['feature'] = train.columns.values
fold_importance_df['importance'] = importances
fold_importance_df['fold'] = i
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 i += 1
print('{} cv RMSE scores : {}'.format(label, cv_scores))
print('{} cv mean RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv std RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv QWK scores : {}'.format(label, qwk_scores))
print('{} cv mean QWK score : {}'.format(label, np.mean(qwk_scores)))
 print('{} cv std QWK score : {}'.format(label, np.std(qwk_scores)))
 pred_full_test = pred_full_test / 5.0
results = {'label': label,
'train': pred_train, 'test': pred_full_test,
'cv': cv_scores, 'qwk': qwk_scores,
'importance': feature_importance_df,
'coefficients': all_coefficients}
return results
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 80,
'max_depth': 9,
'learning_rate': 0.005,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.01,
'min_child_samples': 150,
'min_child_weight': 0.1,
'verbosity': -1,
'data_random_seed': 3,
'early_stop': 100,
'verbose_eval': 100000,
'num_rounds': 1000000}
def runLGB(train_X, train_y, test_X, test_y, test_X2, params):
print('Prep LGB')
 d_train = lgb.Dataset(train_X, label=train_y)
 d_valid = lgb.Dataset(test_X, label=test_y)
 watchlist = [d_train, d_valid]
print('Train LGB')
 num_rounds = params.pop('num_rounds')
 verbose_eval = params.pop('verbose_eval')
 early_stop = None
if params.get('early_stop'):
early_stop = params.pop('early_stop')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 print('Predict 1/2')
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 optR = OptimizedRounder()
optR.fit(pred_test_y, test_y)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(pred_test_y, coefficients)
 print("Valid Counts = ", Counter(test_y))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(test_y, pred_test_y_k)
 print("QWK = ", qwk)
 print('Predict 2/2')
 pred_test_y2 = model.predict(test_X2, num_iteration=model.best_iteration)
 return pred_test_y.reshape(-1, 1), pred_test_y2.reshape(-1, 1), model.feature_importance() , coefficients, qwk
results = run_cv_model(train, test, target, runLGB, params, rmse, 'lgb' )<compute_train_metric>
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID' )
optR = OptimizedRounder()
coefficients_ = np.mean(results['coefficients'], axis=0)
 print(coefficients_)
 train_predictions = [r[0] for r in results['train']]
train_predictions = optR.predict(train_predictions, coefficients_ ).astype(int)
 Counter(train_predictions )<predict_on_test>
for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]
optR = OptimizedRounder()
test_predictions = [r[0] for r in results['test']]
test_predictions = optR.predict(test_predictions, coefficients_ ).astype(int)
 Counter(test_predictions )<create_dataframe>
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('none' )
pd.DataFrame(sk_cmatrix(target, train_predictions), index=list(range(5)) , columns=list(range(5)) )<compute_test_metric>
X_temp['Length_Description'] = X_text['Description'].map(len)
 X_temp['Length_metadata_annots_top_desc'] = X_text['metadata_annots_top_desc'].map(len)
 X_temp['Lengths_sentiment_entities'] = X_text['sentiment_entities'].map(len )
quadratic_weighted_kappa(target, train_predictions)
 rmse(target, [r[0] for r in results['train']])
 submission = pd.DataFrame({'PetID': test_id, 'AdoptionSpeed': test_predictions})
 submission.head()<save_to_csv>
n_components = 16
text_features = []
for i in X_text.columns:
print(f'generating features from: {i}')
 tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_{}_'.format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )
submission.to_csv('submission.csv', index=False )<save_to_csv>
X_temp = X_temp.merge(img_features, how='left', on='PetID' )
submission.to_csv('submission.csv', index=False )<set_options>
train_df_ids = train[['PetID']]
test_df_ids = test[['PetID']]
train_df_imgs = pd.DataFrame(train_image_files)
 train_df_imgs.columns = ['image_filename']
train_imgs_pets = train_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 def getSize(filename):
st = os.stat(filename)
 return st.st_size
def getDimensions(filename):
img_size = Image.open(filename ).size
return img_size
train_df_imgs['image_size'] = train_df_imgs['image_filename'].apply(getSize)
 train_df_imgs['temp_size'] = train_df_imgs['image_filename'].apply(getDimensions)
 train_df_imgs['width'] = train_df_imgs['temp_size'].apply(lambda x : x[0])
 train_df_imgs['height'] = train_df_imgs['temp_size'].apply(lambda x : x[1])
 train_df_imgs = train_df_imgs.drop(['temp_size'], axis=1)
 test_df_imgs['image_size'] = test_df_imgs['image_filename'].apply(getSize)
 test_df_imgs['temp_size'] = test_df_imgs['image_filename'].apply(getDimensions)
 test_df_imgs['width'] = test_df_imgs['temp_size'].apply(lambda x : x[0])
 test_df_imgs['height'] = test_df_imgs['temp_size'].apply(lambda x : x[1])
 test_df_imgs = test_df_imgs.drop(['temp_size'], axis=1)
 aggs = {
'image_size': ['sum', 'mean', 'var'],
'width': ['sum', 'mean', 'var'],
'height': ['sum', 'mean', 'var'],
}
agg_train_imgs = train_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()
agg_test_imgs = test_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index()
agg_imgs = pd.concat([agg_train_imgs, agg_test_imgs], axis=0 ).reset_index(drop=True )
np.random.seed(369 )<compute_test_metric>
X_temp = X_temp.merge(agg_imgs, how='left', on='PetID' )
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<compute_test_metric>
X_temp = X_temp.drop(to_drop_columns, axis=1 )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<compute_test_metric>
X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1 )
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))<load_from_disk>
X_train_non_null.isnull().any().any() , X_test_non_null.isnull().any().any()
def get_sentiment(ids, path):
dic = {
"doc_sent_mag": [],
"doc_sent_score": []
}
for pet in ids:
try:
with open(path + pet + '.json', 'r')as f:
sentiment = json.load(f)
 dic["doc_sent_mag"].append(sentiment['documentSentiment']['magnitude'])
 dic["doc_sent_score"].append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
dic["doc_sent_mag"].append(np.nan)
 dic["doc_sent_score"].append(np.nan)
 return dic
def get_meta(ids, path, n):
dic = {
"vertex_xs": [],
"vertex_ys": [],
"bounding_confidences": [],
"bounding_importance_fracs": [],
"dominant_blues": [],
"dominant_greens": [],
"dominant_reds": [],
"dominant_pixel_fracs": [],
"dominant_scores": [],
"mean_red": [],
"mean_blue": [],
"mean_green": [],
"label_descriptions": [],
"label_scores": []
}
for pet in ids:
try:
with open(path+ pet + '-{}.json'.format(str(n)) , 'r')as f:
data = json.load(f)
 dic["vertex_xs"].append(data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x'])
 dic["vertex_ys"].append(data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y'])
 dic["bounding_confidences"].append(data['cropHintsAnnotation']['cropHints'][0]['confidence'])
 dic["bounding_importance_fracs"].append(data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1))
try:
dic["dominant_blues"].append(data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue'])
 except:
dic["dominant_blues"].append(0)
 try:
dic["dominant_greens"].append(data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green'])
 except:
dic["dominant_greens"].append(0)
 try:
dic["dominant_reds"].append(data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red'])
 except:
dic["dominant_reds"].append(0)
 dic["dominant_pixel_fracs"].append(data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction'])
 dic["dominant_scores"].append(data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score'])

if data.get('labelAnnotations'):
dic["label_descriptions"].append(data['labelAnnotations'][0]['description'])
 dic["label_scores"].append(data['labelAnnotations'][0]['score'])
 else:
dic["label_descriptions"].append('nothing')
 dic["label_scores"].append(np.nan)
 except FileNotFoundError:
dic["vertex_xs"].append(np.nan)
 dic["vertex_ys"].append(np.nan)
 dic["bounding_confidences"].append(np.nan)
 dic["bounding_importance_fracs"].append(np.nan)
 dic["dominant_blues"].append(np.nan)
 dic["dominant_greens"].append(np.nan)
 dic["dominant_reds"].append(np.nan)
 dic["dominant_pixel_fracs"].append(np.nan)
 dic["dominant_scores"].append(np.nan)
 dic["label_descriptions"].append('nothing')
 dic["label_scores"].append(np.nan)
 return dic
def replace_nan(train, test, replace_dic):
for col, value in replace_dic.items() :
train[col] = train[col].replace(value, np.nan)
 test[col] = test[col].replace(value, np.nan)
 return train, test
def get_color(x):
result = [0 for i in range(7)]
for i in range(1, 8):
if i in list(x.values):
result[i-1] = 1
return pd.Series(result )<load_from_csv>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )
%%time
train = pd.read_csv(".. /input/train/train.csv")
 test = pd.read_csv(".. /input/test/test.csv" )<prepare_x_and_y>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']
fe_input_path = ".. /input/"
n_meta = 1
target = train['AdoptionSpeed']
train_id = train['PetID']
test_id = test['PetID']
train.drop(['AdoptionSpeed', 'PetID'], axis=1, inplace=True)
 test.drop(['PetID'], axis=1, inplace=True)
 color_result_col = ["color{}".format(i)for i in range(1, 8)]
color_pick_col = ["Color1", "Color2", "Color3"]
train_color = pd.DataFrame(train[color_pick_col].apply(get_color, axis=1 ).values, columns=color_result_col)
 test_color = pd.DataFrame(test[color_pick_col].apply(get_color, axis=1 ).values, columns=color_result_col)
 train = pd.concat(( train, train_color), axis=1)
 test = pd.concat(( test, test_color), axis=1)
 train.drop(color_pick_col, axis=1, inplace=True)
 test.drop(color_pick_col, axis=1, inplace=True )<feature_engineering>
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'eta': 0.0123,
'subsample': 0.8,
'colsample_bytree': 0.85,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
}
%%time
dic = get_sentiment(train_id, join(fe_input_path, 'train_sentiment/'))
train.loc[:, 'doc_sent_mag'] = dic["doc_sent_mag"]
train.loc[:, 'doc_sent_score'] = dic["doc_sent_score"]
dic = get_sentiment(test_id, join(fe_input_path, 'test_sentiment/'))
test.loc[:, 'doc_sent_mag'] = dic["doc_sent_mag"]
test.loc[:, 'doc_sent_score'] = dic["doc_sent_score"]<feature_engineering>
def run_xgb(params, X_train, X_test):
n_splits = 10
verbose_eval = 1000
num_rounds = 60000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test
for i in range(1, n_meta+1):
dic = get_meta(train_id, join(fe_input_path, 'train_metadata/'), i)
 train.loc[:, 'vertex_x_{}'.format(str(i)) ] = dic["vertex_xs"]
train.loc[:, 'vertex_y_{}'.format(str(i)) ] = dic["vertex_ys"]
train.loc[:, 'bounding_confidence_{}'.format(str(i)) ] = dic["bounding_confidences"]
train.loc[:, 'bounding_importance_{}'.format(str(i)) ] = dic["bounding_importance_fracs"]
train.loc[:, 'dominant_blue_{}'.format(str(i)) ] = dic["dominant_blues"]
train.loc[:, 'dominant_green_{}'.format(str(i)) ] = dic["dominant_greens"]
train.loc[:, 'dominant_red_{}'.format(str(i)) ] = dic["dominant_reds"]
train.loc[:, 'dominant_pixel_frac_{}'.format(str(i)) ] = dic["dominant_pixel_fracs"]
train.loc[:, 'dominant_score_{}'.format(str(i)) ] = dic["dominant_scores"]
train.loc[:, 'label_description_{}'.format(str(i)) ] = dic["label_descriptions"]
train.loc[:, 'label_score_{}'.format(str(i)) ] = dic["label_scores"]
dic = get_meta(test_id, join(fe_input_path, 'test_metadata/'), i)
 test.loc[:, 'vertex_x_{}'.format(str(i)) ] = dic["vertex_xs"]
test.loc[:, 'vertex_y_{}'.format(str(i)) ] = dic["vertex_ys"]
test.loc[:, 'bounding_confidence_{}'.format(str(i)) ] = dic["bounding_confidences"]
test.loc[:, 'bounding_importance_{}'.format(str(i)) ] = dic["bounding_importance_fracs"]
test.loc[:, 'dominant_blue_{}'.format(str(i)) ] = dic["dominant_blues"]
test.loc[:, 'dominant_green_{}'.format(str(i)) ] = dic["dominant_greens"]
test.loc[:, 'dominant_red_{}'.format(str(i)) ] = dic["dominant_reds"]
test.loc[:, 'dominant_pixel_frac_{}'.format(str(i)) ] = dic["dominant_pixel_fracs"]
test.loc[:, 'dominant_score_{}'.format(str(i)) ] = dic["dominant_scores"]
test.loc[:, 'label_description_{}'.format(str(i)) ] = dic["label_descriptions"]
test.loc[:, 'label_score_{}'.format(str(i)) ] = dic["label_scores"]<feature_engineering>
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null )
eng_stopwords = set(stopwords.words("english"))
remove_punctuation_map = dict(( ord(char), ' ')for char in string.punctuation)
 stemmer = nltk.stem.porter.PorterStemmer()
def stem_tokens(tokens):
lst = [stemmer.stem(item)for item in tokens]
return ' '.join(lst)
 def get_textfeats(df, col, flag=True):
df[col] = df[col].fillna('none' ).astype(str)
 df[col] = df[col].str.lower()
df[col] = df[col].apply(lambda x: stem_tokens(nltk.word_tokenize(x.translate(remove_punctuation_map))))
return df
def load_text(train, test):
train = get_textfeats(train, "Description")
 test = get_textfeats(test, "Description")
 train_desc = train['Description'].values
test_desc = test['Description'].values
train_corpus = [text_to_word_sequence(text)for text in tqdm(train_desc)]
test_corpus = [text_to_word_sequence(text)for text in tqdm(test_desc)]
return train_corpus, test_corpus
def get_result(corpus, model):
result = []
for text in corpus:
n_skip = 0
for n_w, word in enumerate(text):
try:
vec_ = model.wv[word]
except:
n_skip += 1
continue
if n_w == 0:
vec = vec_
else:
vec = vec + vec_
vec = vec /(n_w - n_skip + 1)
 result.append(vec)
 return result
train_corpus, test_corpus = load_text(train, test)
 model = word2vec.Word2Vec(train_corpus+test_corpus, size=200, window=10, max_vocab_size=50000, seed=0)
 train_result = get_result(train_corpus, model)
 test_result = get_result(test_corpus, model)
 w2v_cols = ["wv{}".format(i)for i in range(1, 201)]
train_result = pd.DataFrame(train_result)
 train_result.columns = w2v_cols
test_result = pd.DataFrame(test_result)
 test_result.columns = w2v_cols
train = pd.concat(( train, train_result), axis=1)
 test = pd.concat(( test, test_result), axis=1 )<drop_column>
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk )
%%time
train.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True)
 test.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True )<data_type_conversions>
coefficients_ = coefficients.copy()
coefficients_[0] = 1.66
coefficients_[1] = 2.13
coefficients_[3] = 2.85
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}' )
replace_dic = {
"Gender": 3,
"MaturitySize": 0,
"FurLength": 0,
"MaturitySize": 0,
"Vaccinated": 3,
"Dewormed": 3,
"Sterilized": 3,
"Health": 0
}
train, test = replace_nan(train, test, replace_dic)
 cat_cols = ['Type', 'Breed1', 'Breed2', 'Gender', 'MaturitySize',
'FurLength', 'Vaccinated', 'Dewormed', 'Sterilized', 'Health',
'State', 'label_description_1']
train.loc[:, cat_cols] = train[cat_cols].astype('category')
 test.loc[:, cat_cols] = test[cat_cols].astype('category')
 print(train.shape)
 print(test.shape)
 train.head()<find_best_model_class>
Counter(train_predictions )
def run_cv_model(train, test, target, model_fn, params={}, eval_fn=None, label='model'):
kf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)
 fold_splits = kf.split(train, target)
 cv_scores = []
qwk_scores = []
pred_full_test = 0
pred_train = np.zeros(( train.shape[0], 5))
all_coefficients = np.zeros(( 5, 4))
feature_importance_df = pd.DataFrame()
i = 1
for dev_index, val_index in fold_splits:
print('Started ' + label + ' fold ' + str(i)+ '/5')
 if isinstance(train, pd.DataFrame):
dev_X, val_X = train.iloc[dev_index], train.iloc[val_index]
dev_y, val_y = target[dev_index], target[val_index]
else:
dev_X, val_X = train[dev_index], train[val_index]
dev_y, val_y = target[dev_index], target[val_index]
params2 = params.copy()
pred_val_y, pred_test_y, importances, coefficients, qwk = model_fn(dev_X, dev_y, val_X, val_y, test, params2)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index] = pred_val_y
all_coefficients[i-1, :] = coefficients
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 qwk_scores.append(qwk)
 print(label + ' cv score {}: RMSE {} QWK {}'.format(i, cv_score, qwk))
fold_importance_df = pd.DataFrame()
fold_importance_df['feature'] = train.columns.values
fold_importance_df['importance'] = importances
fold_importance_df['fold'] = i
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 i += 1
print('{} cv RMSE scores : {}'.format(label, cv_scores))
print('{} cv mean RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv std RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv QWK scores : {}'.format(label, qwk_scores))
print('{} cv mean QWK score : {}'.format(label, np.mean(qwk_scores)))
 print('{} cv std QWK score : {}'.format(label, np.std(qwk_scores)))
 pred_full_test = pred_full_test / 5.0
results = {'label': label,
'train': pred_train, 'test': pred_full_test,
'cv': cv_scores, 'qwk': qwk_scores,
'importance': feature_importance_df,
'coefficients': all_coefficients}
return results
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 80,
'max_depth': 9,
'learning_rate': 0.01,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.01,
'min_child_samples': 150,
'min_child_weight': 0.1,
'verbosity': -1,
'data_random_seed': 3,
'early_stop': 100,
'verbose_eval': 100,
'num_rounds': 10000}
def runLGB(train_X, train_y, test_X, test_y, test_X2, params):
print('Prep LGB')
 d_train = lgb.Dataset(train_X, label=train_y)
 d_valid = lgb.Dataset(test_X, label=test_y)
 watchlist = [d_train, d_valid]
print('Train LGB')
 num_rounds = params.pop('num_rounds')
 verbose_eval = params.pop('verbose_eval')
 early_stop = None
if params.get('early_stop'):
early_stop = params.pop('early_stop')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 print('Predict 1/2')
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 optR = OptimizedRounder()
optR.fit(pred_test_y, test_y)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(pred_test_y, coefficients)
 print("Valid Counts = ", Counter(test_y))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(test_y, pred_test_y_k)
 print("QWK = ", qwk)
 print('Predict 2/2')
 pred_test_y2 = model.predict(test_X2, num_iteration=model.best_iteration)
 return pred_test_y.reshape(-1, 1), pred_test_y2.reshape(-1, 1), model.feature_importance() , coefficients, qwk
results = run_cv_model(train, test, target, runLGB, params, rmse, 'lgb' )<compute_train_metric>
Counter(test_predictions )
optR = OptimizedRounder()
coefficients_ = np.mean(results['coefficients'], axis=0)
 print(coefficients_)
 train_predictions = [r[0] for r in results['train']]
train_predictions = optR.predict(train_predictions, coefficients_ ).astype(int)
 Counter(train_predictions )<predict_on_test>
submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions})
 submission.to_csv('submission.csv', index=False)
 submission.head()