Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<import_modules>
np.random.seed(369 )
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from keras.utils import np_utils
from keras.models import Sequential
from keras.layers import Dense, Conv2D , MaxPool2D , Flatten , Dropout , BatchNormalization
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler<load_from_csv>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )
train = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 test = pd.read_csv("/kaggle/input/digit-recognizer/test.csv" )<prepare_x_and_y>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))
train_Y = np.array(train["label"])
 train_Y = np_utils.to_categorical(train_Y, num_classes = 10 )<prepare_x_and_y>
train_df = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 img_size = 256
batch_size = 16
train_X = np.array(train.iloc[:, 1:])
 test_X = np.array(test )<split>
pet_ids = train_df['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
train_X, valid_X, train_Y, valid_Y = train_test_split(train_X, train_Y, shuffle=True, test_size = 0.1, random_state=2, stratify=train_Y )<init_hyperparams>
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights=".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(train_X )<choose_model_class>
features = {}
for b in tqdm_notebook(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
nets = 5
model = [0] *nets
for i in range(nets):
model[i] = Sequential()
model[i].add(Conv2D(32, kernel_size = 3, activation='relu', input_shape =(28, 28, 1)))
 model[i].add(BatchNormalization())
 model[i].add(Conv2D(32, kernel_size = 3, activation='relu'))
model[i].add(BatchNormalization())
 model[i].add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))
model[i].add(BatchNormalization())
 model[i].add(Dropout(0.4))
model[i].add(Conv2D(64, kernel_size = 3, activation='relu'))
model[i].add(BatchNormalization())
 model[i].add(Conv2D(64, kernel_size = 3, activation='relu'))
model[i].add(BatchNormalization())
 model[i].add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))
model[i].add(BatchNormalization())
 model[i].add(Dropout(0.4))
model[i].add(Conv2D(128, kernel_size = 4, activation='relu'))
model[i].add(BatchNormalization())
 model[i].add(Flatten())
 model[i].add(Dropout(0.4))
model[i].add(Dense(10, activation='softmax'))
model[i].compile(optimizer="rmsprop", loss="categorical_crossentropy", metrics=["accuracy"])
<train_model>
train_feats = pd.DataFrame.from_dict(features, orient='index')
 train_feats.columns = ['pic_'+str(i)for i in range(train_feats.shape[1])]
annealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)
 history = [0] * nets
epochs = 45
for j in range(nets):
print("CNN ",j+1)
 history[j] = model[j].fit_generator(datagen.flow(train_X, train_Y, batch_size=64),
epochs = epochs, steps_per_epoch = train_X.shape[0]//64,
validation_data =(valid_X, valid_Y), callbacks=[annealer], verbose=1)
 print("CNN {0:d} DONE".format(j+1))<save_to_csv>
test_df = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 pet_ids = test_df['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm_notebook(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/test_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
results = np.zeros(( test_X.shape[0],10))
for j in range(nets):
results = results + model[j].predict(test_X)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label")
 submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("submission.csv",index=False )<set_options>
test_feats = pd.DataFrame.from_dict(features, orient='index')
 test_feats.columns = ['pic_'+str(i)for i in range(test_feats.shape[1])]
warnings.filterwarnings("ignore" )<set_options>
test_feats = test_feats.reset_index()
test_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 train_feats = train_feats.reset_index()
train_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 test_feats.head()
%%javascript
IPython.OutputArea.prototype._should_scroll = function() {
return False;
}<install_modules>
print('Train')
 train = pd.read_csv(".. /input/petfinder-adoption-prediction/train/train.csv")
 print(train.shape)
 print('Test')
 test = pd.read_csv(".. /input/petfinder-adoption-prediction/test/test.csv")
 print(test.shape)
 print('Breeds')
 breeds = pd.read_csv(".. /input/petfinder-adoption-prediction/breed_labels.csv")
 print(breeds.shape)
 print('Colors')
 colors = pd.read_csv(".. /input/petfinder-adoption-prediction/color_labels.csv")
 print(colors.shape)
 print('States')
 states = pd.read_csv(".. /input/petfinder-adoption-prediction/state_labels.csv")
 print(states.shape)
 target = train['AdoptionSpeed']
train_id = train['PetID']
test_id = test['PetID']
train = pd.merge(train, train_feats, how='left', on='PetID')
 test = pd.merge(test, test_feats, how='left', on='PetID')
 train.drop(['AdoptionSpeed', 'PetID'], axis=1, inplace=True)
 test.drop(['PetID'], axis=1, inplace=True )
!apt-get install p7zip<load_from_csv>
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in train_id:
try:
with open('.. /input/petfinder-adoption-prediction/train_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 train.loc[:, 'doc_sent_mag'] = doc_sent_mag
train.loc[:, 'doc_sent_score'] = doc_sent_score
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in test_id:
try:
with open('.. /input/petfinder-adoption-prediction/test_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 test.loc[:, 'doc_sent_mag'] = doc_sent_mag
test.loc[:, 'doc_sent_score'] = doc_sent_score
train_data = pd.read_csv("train.tsv", sep='\t')
 test_data = pd.read_csv(".. /input/mercari-price-suggestion-challenge/test_stg2.tsv.zip" , sep='\t' )<create_dataframe>
train_desc = train.Description.fillna("none" ).values
test_desc = test.Description.fillna("none" ).values
tfv = TfidfVectorizer(min_df=3, max_features=10000,
strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1,
stop_words = 'english')
 tfv.fit(list(train_desc))
X = tfv.transform(train_desc)
 X_test = tfv.transform(test_desc)
 print("X(tfidf):", X.shape)
 svd = TruncatedSVD(n_components=200)
 svd.fit(X)
 X = svd.transform(X)
 print("X(svd):", X.shape)
 X = pd.DataFrame(X, columns=['svd_{}'.format(i)for i in range(200)])
 train = pd.concat(( train, X), axis=1)
 X_test = svd.transform(X_test)
 X_test = pd.DataFrame(X_test, columns=['svd_{}'.format(i)for i in range(200)])
 test = pd.concat(( test, X_test), axis=1)
 print("train:", train.shape)
train_copy = train_data.copy()<set_options>
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in train_id:
try:
with open('.. /input/petfinder-adoption-prediction/train_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 print(nl_count)
 train.loc[:, 'vertex_x'] = vertex_xs
train.loc[:, 'vertex_y'] = vertex_ys
train.loc[:, 'bounding_confidence'] = bounding_confidences
train.loc[:, 'bounding_importance'] = bounding_importance_fracs
train.loc[:, 'dominant_blue'] = dominant_blues
train.loc[:, 'dominant_green'] = dominant_greens
train.loc[:, 'dominant_red'] = dominant_reds
train.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
train.loc[:, 'dominant_score'] = dominant_scores
train.loc[:, 'label_description'] = label_descriptions
train.loc[:, 'label_score'] = label_scores
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in test_id:
try:
with open('.. /input/petfinder-adoption-prediction/test_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 test.loc[:, 'vertex_x'] = vertex_xs
test.loc[:, 'vertex_y'] = vertex_ys
test.loc[:, 'bounding_confidence'] = bounding_confidences
test.loc[:, 'bounding_importance'] = bounding_importance_fracs
test.loc[:, 'dominant_blue'] = dominant_blues
test.loc[:, 'dominant_green'] = dominant_greens
test.loc[:, 'dominant_red'] = dominant_reds
test.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
test.loc[:, 'dominant_score'] = dominant_scores
test.loc[:, 'label_description'] = label_descriptions
test.loc[:, 'label_score'] = label_scores
gc.collect()<string_transform>
train.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True)
 test.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True )
def split_cat(text):
try: return text.split("/")
 except: return("No Label", "No Label", "No Label" )<feature_engineering>
numeric_cols = ['Age', 'Quantity', 'Fee', 'VideoAmt', 'PhotoAmt', 'AdoptionSpeed',
'doc_sent_mag', 'doc_sent_score', 'dominant_score', 'dominant_pixel_frac',
'dominant_red', 'dominant_green', 'dominant_blue', 'bounding_importance',
'bounding_confidence', 'vertex_x', 'vertex_y', 'label_score'] +\
[col for col in train.columns if col.startswith('pic')or col.startswith('svd')]
cat_cols = list(set(train.columns)- set(numeric_cols))
train.loc[:, cat_cols] = train[cat_cols].astype('category')
 test.loc[:, cat_cols] = test[cat_cols].astype('category')
 print(train.shape)
 print(test.shape)
 foo = train.dtypes
cat_feature_names = foo[foo == "category"]
cat_features = [train.columns.get_loc(c)for c in train.columns if c in cat_feature_names]
train_copy['main_category'], train_copy['subcat_1'], train_copy['subcat_2'] = zip(*train_copy['category_name'].apply(lambda x: split_cat(x)) )<categorify>
train = train.drop('label_description',axis=1)
 test= test.drop('label_description', axis=1 )
def toNumeric(data,to):
if train_copy[data].dtype == type(object):
le = preprocessing.LabelEncoder()
train_copy[to] = le.fit_transform(train_copy[data].astype(str))
toNumeric('name','n_name')
 toNumeric('category_name','n_category_name')
 toNumeric('brand_name','n_brand_name')
 toNumeric('main_category','n_main_category')
 toNumeric('subcat_1','n_subcat_1')
 toNumeric('subcat_2','n_subcat_2')
 train_copy.head()<count_missing_values>
col_vals_dict= {col: list(train[col].unique())for col in train.select_dtypes(['category'])}
embed_cols = []
for c in col_vals_dict:
if len(col_vals_dict[c])>=2:
embed_cols.append(c)
 print(c + ': %d values' % len(col_vals_dict[c]))
train_copy.isnull().any()<categorify>
class CyclicLR(Callback):

 def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1/(2.**(x-1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma**(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1+self.clr_iterations/(2*self.step_size))
x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr-self.base_lr)*np.maximum(0,(1-x)) *self.scale_fn(self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
self.history.setdefault('lr', [] ).append(K.get_value(self.model.optimizer.lr))
self.history.setdefault('iterations', [] ).append(self.trn_iterations)
 for k, v in logs.items() :
self.history.setdefault(k, [] ).append(v)
 K.set_value(self.model.optimizer.lr, self.clr())
 class QWKEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.history = []
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, batch_size=1000, verbose=0)
 y_pred = eval_predict(self.y_val, y_pred)
 score = quadratic_weighted_kappa(self.y_val, y_pred)
 print("QWK - epoch: %d - score: %.6f
" %(epoch+1, score))
self.history.append(score)
 if score >= max(self.history): self.model.save('checkpoint.h5')
 def eval_predict(y=[], y_pred=[], coeffs=None, ret_coeffs=False):
optR = OptimizedRounder()
if not coeffs:
optR.fit(y_pred.reshape(-1,), y)
 coeffs = optR.coefficients()
if ret_coeffs: return optR.coefficients()
return optR.predict(y_pred, coeffs ).reshape(-1, )
def fill_missing_data(data):
data.category_name.fillna(value = "Other/Other/Other", inplace = True)
 data.brand_name.fillna(value = "Unknown brand", inplace = True)
 data.item_description.fillna(value = "No description", inplace = True)
 return data<correct_missing_values>
seed = 2019
np.random.seed(seed)
 tf.set_random_seed(seed)
 def get_model(dr=0.15, seed=2019):
np.random.seed(seed)
 tf.set_random_seed(seed)
 sess = tf.Session()
K.set_session(sess)
 inputs = []
embeddings = []
input_type = Input(shape=[1], name='Type')
 embedding = Embedding(2,1, input_length=1 )(input_type)
 embedding = Reshape(target_shape=(1,))(embedding)
 inputs.append(input_type)
 embeddings.append(embedding)
 input_breed1 = Input(shape=[1],name= 'Breed1')
 embedding = Embedding(176,10, input_length=1 )(input_breed1)
 embedding = Reshape(target_shape=(10,))(embedding)
 inputs.append(input_breed1)
 embeddings.append(embedding)
 input_breed2 = Input(shape=[1], name= 'Breed2')
 embedding = Embedding(135,10, input_length=1 )(input_breed2)
 embedding = Reshape(target_shape=(10,))(embedding)
 inputs.append(input_breed2)
 embeddings.append(embedding)
 input_gender = Input(shape=[1], name= 'Gender')
 embedding = Embedding(3,2, input_length=1 )(input_gender)
 embedding = Reshape(target_shape=(2,))(embedding)
 inputs.append(input_gender)
 embeddings.append(embedding)
 input_color1 = Input(shape=[1], name= 'Color1')
 embedding = Embedding(7,4, input_length=1 )(input_color1)
 embedding = Reshape(target_shape=(4,))(embedding)
 inputs.append(input_color1)
 embeddings.append(embedding)
 input_color2 = Input(shape=[1], name= 'Color2')
 embedding = Embedding(7,4, input_length=1 )(input_color2)
 embedding = Reshape(target_shape=(4,))(embedding)
 inputs.append(input_color2)
 embeddings.append(embedding)
 input_color3 = Input(shape=[1], name= 'Color3')
 embedding = Embedding(6,3, input_length=1 )(input_color3)
 embedding = Reshape(target_shape=(3,))(embedding)
 inputs.append(input_color3)
 embeddings.append(embedding)
 input_maturity_size = Input(shape=[1], name= 'MaturitySize')
 embedding = Embedding(4,2, input_length=1 )(input_maturity_size)
 embedding = Reshape(target_shape=(2,))(embedding)
 inputs.append(input_maturity_size)
 embeddings.append(embedding)
 input_fur_length = Input(shape=[1], name= 'FurLength')
 embedding = Embedding(3,2, input_length=1 )(input_fur_length)
 embedding = Reshape(target_shape=(2,))(embedding)
 inputs.append(input_fur_length)
 embeddings.append(embedding)
 input_vaccinated = Input(shape=[1], name= 'Vaccinated')
 embedding = Embedding(3,2, input_length=1 )(input_vaccinated)
 embedding = Reshape(target_shape=(2,))(embedding)
 inputs.append(input_vaccinated)
 embeddings.append(embedding)
 input_dewormed = Input(shape=[1], name='Dewormed')
 embedding = Embedding(3,2, input_length=1 )(input_vaccinated)
 embedding = Reshape(target_shape=(2,))(embedding)
 inputs.append(input_dewormed)
 embeddings.append(embedding)
 input_sterilized = Input(shape=[1], name='Sterilized')
 embedding = Embedding(3,2, input_length=1 )(input_sterilized)
 embedding = Reshape(target_shape=(2,))(embedding)
 inputs.append(input_sterilized)
 embeddings.append(embedding)
 input_health = Input(shape=[1], name='Health')
 embedding = Embedding(3,2, input_length=1 )(input_health)
 embedding = Reshape(target_shape=(2,))(embedding)
 inputs.append(input_health)
 embeddings.append(embedding)
 input_state = Input(shape=[1], name='State')
 embedding = Embedding(14,7, input_length=1 )(input_health)
 embedding = Reshape(target_shape=(7,))(embedding)
 inputs.append(input_state)
 embeddings.append(embedding)
 input_pic = Input(shape=[num_train[pic_cols].shape[1]], name='pic_col')
 bn_pic = BatchNormalization()(input_pic)
 inputs.append(input_pic)
 embeddings.append(bn_pic)
 input_svd = Input(shape=[num_train[svd_cols].shape[1]], name='svd_col')
 bn_svd = BatchNormalization()(input_svd)
 inputs.append(input_svd)
 embeddings.append(bn_svd)
 input_meta = Input(shape=[num_train[meta_cols].shape[1]], name='meta_col')
 bn_meta = BatchNormalization()(input_meta)
 inputs.append(input_meta)
 embeddings.append(bn_meta)
 input_sent = Input(shape=[num_train[sent_cols].shape[1]], name='sent_col')
 bn_sent = BatchNormalization()(input_sent)
 inputs.append(input_sent)
 embeddings.append(bn_sent)
 input_other = Input(shape=[num_train[othernum_cols].shape[1]], name='other_col')
 bn_other = BatchNormalization()(input_other)
 inputs.append(input_other)
 embeddings.append(bn_other)
 main_l= Concatenate()(embeddings)
 main_l = BatchNormalization()(main_l)
 main_l= Dropout(dr )(Dense(250, activation = 'relu' )(main_l))
main_l= Dropout(dr )(Dense(100, activation = 'relu' )(main_l))
main_l = Dense(50, activation='relu' )(main_l)
 main_l= Dense(10, activation = 'relu' )(main_l)
 output = Dense(1, activation = 'linear' )(main_l)
 model = Model(inputs, output)
 optimizer= optimizers.Adam()
model.compile(loss='logcosh', optimizer=optimizer)
 return model, sess
train_copy = fill_missing_data(train_copy)
 train_copy = train_copy.dropna()
print(np.shape(train_copy))
train_copy.head()<set_options>
def preproc(df):
input_list = []
for c in embed_cols:
raw_vals = np.unique(df[c])
 val_map = {}
for i in range(len(raw_vals)) :
val_map[raw_vals[i]] = i
input_list.append(df[c].map(val_map ).fillna(0 ).values)
 input_list.append(df[pic_cols].values)
 input_list.append(df[svd_cols].values)
 input_list.append(df[meta_cols].values)
 input_list.append(df[sent_cols].values)
 input_list.append(df[othernum_cols].values)
 return input_list
sns.set(context='notebook')
 sns.set_style("whitegrid", {'axes.grid' : False})
 plt.tight_layout()<feature_engineering>
epochs = 5
BATCH_SIZE = 1000
N_SPLITS = 5
model, sess = get_model()
K.set_session(sess)
 def run_cv_model(train, target, model_fn, eval_fn=None, label='model'):
kf = StratifiedKFold(n_splits=N_SPLITS, random_state=1906, shuffle=True)
 fold_splits = kf.split(train, target)
 cv_scores = []
qwk_scores = []
pred_full_test = 0
pred_train = np.zeros(( train.shape[0],))
all_coefficients = np.zeros(( N_SPLITS, 4))
feature_importance_df = pd.DataFrame()
i = 1
for train_index, val_index in fold_splits:
print('Started ' + label + ' fold ' + str(i)+ '/' + str(N_SPLITS))
X_train = preproc(train.iloc[train_index])
 X_val = preproc(train.iloc[val_index])
 y_train = target.iloc[train_index].values
y_val = target.iloc[val_index].values
X_test = preproc(test.copy())
 pred_val_y, pred_test_y, coefficients, qwk = model_fn(X_train, X_val, y_train, y_val,X_test)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index] = pred_val_y
all_coefficients[i-1, :] = coefficients
if eval_fn is not None:
cv_score = eval_fn(y_val, pred_val_y)
 cv_scores.append(cv_score)
 qwk_scores.append(qwk)
 print(label + ' cv score {}: RMSE {} QWK {}'.format(i, cv_score, qwk))
i += 1
print('{} cv RMSE scores : {}'.format(label, cv_scores))
print('{} cv mean RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv std RMSE score : {}'.format(label, np.std(cv_scores)))
 print('{} cv QWK scores : {}'.format(label, qwk_scores))
print('{} cv mean QWK score : {}'.format(label, np.mean(qwk_scores)))
 print('{} cv std QWK score : {}'.format(label, np.std(qwk_scores)))
 pred_full_test = pred_full_test / float(N_SPLITS)
 results = {'label': label,
'train': pred_train,'cv': cv_scores, 'qwk': qwk_scores,'test': pred_full_test,
'coefficients': all_coefficients}
return results
def runNN(X_train, X_val, y_train, y_val, X_test):
model, sess = get_model()
K.set_session(sess)
 clr_tri = CyclicLR(base_lr=2e-3, max_lr=4e-2, step_size=len(X_train)//batch_size, mode="triangular2")
 qwk_eval = QWKEvaluation(validation_data=(X_val, y_val), interval=1)
 history = model.fit(X_train, y_train, epochs=epochs, batch_size=BATCH_SIZE, validation_data=(X_val, y_val),callbacks=[clr_tri, qwk_eval])
 print('Predict 1/2')
 y_pred = model.predict(X_val,batch_size=BATCH_SIZE ).ravel()
optR = OptimizedRounder()
optR.fit(y_pred, y_val)
 coefficients = optR.coefficients()
y_pred_k = optR.predict(y_pred, coefficients)
 print("Valid Counts = ", Counter(y_val))
print("Predicted Counts = ", Counter(y_pred_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(y_val, y_pred_k)
 print("QWK = ", qwk)
 print('Predict 2/2')
 test_pred = model.predict(X_test,batch_size=BATCH_SIZE ).ravel()
return y_pred, test_pred, coefficients, qwk
results = run_cv_model(train, target, runNN, rmse, 'NN' )
train_copy['price'] = np.log1p(train_copy['price'] )<set_options>
optR = OptimizedRounder()
coefficients_ = np.mean(results['coefficients'], axis=0)
 print(coefficients_)
gc.collect()<split>
coefficients_[0] = 1.645
coefficients_[1] = 2.115
coefficients_[3] = 2.84
train_predictions = results['train']
train_predictions = optR.predict(train_predictions, coefficients_ ).astype(int)
 Counter(train_predictions)
 print(quadratic_weighted_kappa(train_predictions,target))
r_data= train_copy[['item_condition_id','shipping','n_name','n_brand_name','n_main_category','n_subcat_1','n_subcat_2']]
X_train, x_test, Y_train, y_test = train_test_split(r_data, train_copy['price'], test_size=0.25, random_state=12345)
 def run_model(model, X_train, Y_train, x_test, y_test, verbose = False):
Y_train = Y_train[:, np.newaxis].ravel()
model.fit(X_train, Y_train)
 y_predict = model.predict(x_test)
 mse = mean_squared_error(y_test,y_predict)
 r_sq = r2_score(y_test,y_predict)
 print("Mean Squared Error Value : "+"{:.2f}".format(mse))
print("R-Squared Value : "+"{:.2f}".format(r_sq))
return model, mse, r_sq<train_on_grid>
optR = OptimizedRounder()
coefficients_ = np.mean(results['coefficients'], axis=0)
 print(coefficients_)
 coefficients_[0] = 1.645
coefficients_[1] = 2.115
coefficients_[3] = 2.84
test_predictions = results['test']
test_predictions = optR.predict(test_predictions, coefficients_ ).astype(int)
 Counter(test_predictions )
ridge_reg = linear_model.Ridge()
print("Ridge Regression")
 print("----------------")
 model_1, mse_1, r_sq_1 = run_model(ridge_reg, X_train, Y_train, x_test, y_test )<train_model>
print("True Distribution:")
 print(pd.value_counts(target, normalize=True ).sort_index())
 print("Test Predicted Distribution:")
 print(pd.value_counts(test_predictions, normalize=True ).sort_index())
 print("Train Predicted Distribution:")
 print(pd.value_counts(train_predictions, normalize=True ).sort_index() )
lgbm_reg = lightgbm.LGBMRegressor()
print("LGBM Regression")
 print("---------------")
 model_2, mse_2, r_sq_2 = run_model(lgbm_reg, X_train, Y_train, x_test, y_test )<train_model>
pd.DataFrame(sk_cmatrix(target, train_predictions), index=list(range(5)) , columns=list(range(5)) )
xgb_params = {'n_estimators':500, 'max_depth':8}
xgb_reg = xgboost.XGBRegressor(**xgb_params)
 print("XGBoost Regression")
 print("------------------")
 model_3, mse_3, r_sq_3 = run_model(xgb_reg, X_train, Y_train, x_test, y_test )<define_variables>
quadratic_weighted_kappa(target, train_predictions)
 rmse(target, results['train'])
 submission = pd.DataFrame({'PetID': test_id, 'AdoptionSpeed': test_predictions})
 submission.head()
combined_data = pd.concat([train_data,test_data])
 train_size = len(train_data)
 submission = test_data[['test_id']]<define_variables>
submission.to_csv('submission.csv', index=False )
<import_modules><EOS> 
submission.to_csv('submission.csv', index=False )