Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<define_variables>
%matplotlib inline
np.random.seed(seed=1337)
 warnings.filterwarnings('ignore')
 split_char = '/'
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<choose_model_class>
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv' )
early_stopping_cb = keras.callbacks.EarlyStopping(patience=20, restore_best_weights=True)
 model_checkpoint_cb = keras.callbacks.ModelCheckpoint("best_mnist_model.h5", save_best_only=True)
 learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc',
patience=5,
verbose=1,
factor=0.5,
min_lr=0.00001)
 history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,Y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[early_stopping_cb, model_checkpoint_cb, learning_rate_reduction] )<train_model>
import cv2
import os
from keras.applications.densenet import preprocess_input, DenseNet121
model = keras.models.load_model("best_mnist_model.h5")
 model.evaluate(X_train, Y_train)
<predict_on_test>
img_size = 256
batch_size = 256
results = model.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )<save_to_csv>
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights=".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_with_aug.csv",index=False )<set_options>
pet_ids = train['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
%matplotlib inline
np.random.seed(42)
<load_from_csv>
train_feats = pd.DataFrame.from_dict(features, orient='index')
 train_feats.columns = [f'pic_{i}' for i in range(train_feats.shape[1])]
train_data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test_data = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
 submission=pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv' )<define_variables>
pet_ids = test['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/test_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
train_data=".. /input/digit-recognizer/train.csv"
test_data = ".. /input/digit-recognizer/test.csv"<split>
test_feats = pd.DataFrame.from_dict(features, orient='index')
 test_feats.columns = [f'pic_{i}' for i in range(test_feats.shape[1])]
Xtrain,Xtest,ytrain,ytest=train_test_split(raw_data[:,1:], raw_data[:,0],test_size=0.2,
stratify=raw_data[:,0],
random_state=42)
<feature_engineering>
train_feats = train_feats.reset_index()
train_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 test_feats = test_feats.reset_index()
test_feats.rename({'index': 'PetID'}, axis='columns', inplace=True )
Xtrain=Xtrain/255.
 Xtest=Xtest/255 .<categorify>
all_ids = pd.concat([train, test], axis=0, ignore_index=True, sort=False)[['PetID']]
all_ids.shape
ytrain= to_categorical(ytrain, num_classes = 10)
 ytest= to_categorical(ytest, num_classes = 10 )<data_type_conversions>
n_components = 32
svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 features_df = pd.concat([train_feats, test_feats], axis=0)
 features = features_df[[f'pic_{i}' for i in range(256)]].values
svd_col = svd_.fit_transform(features)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('IMG_SVD_')
 img_features = pd.concat([all_ids, svd_col], axis=1 )
Xtrain=Xtrain.astype("float32" ).reshape(-1,28,28,1)
 Xtest=Xtest.astype("float32" ).reshape(-1,28,28,1 )<choose_model_class>
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )
train_gen=ImageDataGenerator(rotation_range=20,
width_shift_range=0.1,
height_shift_range=0.1,
zoom_range=0.1,
shear_range=0.2
)<choose_model_class>
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print(f'num of train images files: {len(train_image_files)}')
 print(f'num of train metadata files: {len(train_metadata_files)}')
 print(f'num of train sentiment files: {len(train_sentiment_files)}')
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print(f'num of test images files: {len(test_image_files)}')
 print(f'num of test metadata files: {len(test_metadata_files)}')
 print(f'num of test sentiment files: {len(test_sentiment_files)}' )
def model() :
model=tf.keras.Sequential()
model.add(tf.keras.layers.Conv2D(16,(3,3), activation="relu",padding="SAME",input_shape=(28,28,1)))
 model.add(BatchNormalization())
 model.add(tf.keras.layers.Conv2D(32,(3,3),activation="relu",padding="SAME"))
model.add(BatchNormalization())
 model.add(tf.keras.layers.MaxPool2D(strides=(2,2)))
 model.add(Dropout(0.25))
model.add(tf.keras.layers.Conv2D(32,(3,3),padding='SAME',activation='relu'))
model.add(BatchNormalization())
 model.add(tf.keras.layers.Conv2D(64,(3,3),padding='SAME',activation='relu'))
model.add(BatchNormalization())
 model.add(tf.keras.layers.MaxPool2D(strides=(2,2)))
 model.add(Dropout(0.25))
model.add(tf.keras.layers.Flatten())
 model.add(tf.keras.layers.Dense(512,activation="relu"))
model.add(Dropout(0.25))
model.add(tf.keras.layers.Dense(256,activation="relu"))
model.add(Dropout(0.5))
model.add(tf.keras.layers.Dense(10,activation="softmax"))
return model<choose_model_class>
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_json_file(self, filename):
with open(filename, 'r', encoding='utf-8')as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns')
 file_sentences_sentiment_df = pd.DataFrame(
 {
'magnitude_sum': file_sentences_sentiment['magnitude'].sum(axis=0),
'score_sum': file_sentences_sentiment['score'].sum(axis=0),
'magnitude_mean': file_sentences_sentiment['magnitude'].mean(axis=0),
'score_mean': file_sentences_sentiment['score'].mean(axis=0),
'magnitude_var': file_sentences_sentiment['magnitude'].var(axis=0),
'score_var': file_sentences_sentiment['score'].var(axis=0),
}, index=[0] 
)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
df_sentiment = pd.concat([df_sentiment, file_sentences_sentiment_df], axis=1)
 df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = f'.. /input/petfinder-adoption-prediction/{mode}_sentiment/{pet_id}.json'
try:
sentiment_file = pet_parser.open_json_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob(f'.. /input/petfinder-adoption-prediction/{mode}_metadata/{pet_id}*.json'))
if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_json_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()
annealer = LearningRateScheduler(lambda x: 1e-3 * 0.9 ** x )<choose_model_class>
aggregates = ['sum', 'mean', 'var']
sent_agg = ['sum']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 train_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 test_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()
model.compile(optimizer=Adam(lr=1e-4),loss='categorical_crossentropy',
metrics=['accuracy'] )<train_model>
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]
class MyCallback(tf.keras.callbacks.Callback):
def on_epoch_end(self,epochs,logs={}):
if(logs.get('accuracy')>0.99):
print()
print("Stopping Training")
 self.model.stop_training=True<train_model>
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )
history=model.fit_generator(train_gen.flow(Xtrain,ytrain,batch_size=16),
validation_data=(Xtest[:1000,:], ytest[:1000,:]),
epochs=30,callbacks=[annealer], steps_per_epoch=500
)<compute_test_metric>
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False )
model.evaluate(Xtest,ytest )<data_type_conversions>
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID']
testing = np.loadtxt(test_data, skiprows=1, dtype='int', delimiter=',')
 test = testing.astype("float32")
 test = testing.reshape(-1, 28, 28, 1)/255 .<predict_on_test>
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID' )
ypred=np.argmax(model.predict(test),axis=-1 )<save_to_csv>
for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("MNIST_digit_recog.csv",index=False )<set_options>
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('none' )
%matplotlib inline
np.random.seed(2)
 sns.set(style='white', context='notebook', palette='deep')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
X_temp['Length_Description'] = X_text['Description'].map(len)
 X_temp['Length_metadata_annots_top_desc'] = X_text['metadata_annots_top_desc'].map(len)
 X_temp['Lengths_sentiment_entities'] = X_text['sentiment_entities'].map(len )
train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test = pd.read_csv('.. /input/digit-recognizer/test.csv' )<prepare_x_and_y>
n_components = 16
text_features = []
for i in X_text.columns:
print(f'generating features from: {i}')
 tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_{}_'.format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )
y_train = train.label
x_train = train.drop('label', 1 )<count_missing_values>
X_temp = X_temp.merge(img_features, how='left', on='PetID' )
train.isnull().values.any()<feature_engineering>
train_df_ids = train[['PetID']]
test_df_ids = test[['PetID']]
train_df_imgs = pd.DataFrame(train_image_files)
 train_df_imgs.columns = ['image_filename']
train_imgs_pets = train_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 def getSize(filename):
st = os.stat(filename)
 return st.st_size
def getDimensions(filename):
img_size = Image.open(filename ).size
return img_size
train_df_imgs['image_size'] = train_df_imgs['image_filename'].apply(getSize)
 train_df_imgs['temp_size'] = train_df_imgs['image_filename'].apply(getDimensions)
 train_df_imgs['width'] = train_df_imgs['temp_size'].apply(lambda x : x[0])
 train_df_imgs['height'] = train_df_imgs['temp_size'].apply(lambda x : x[1])
 train_df_imgs = train_df_imgs.drop(['temp_size'], axis=1)
 test_df_imgs['image_size'] = test_df_imgs['image_filename'].apply(getSize)
 test_df_imgs['temp_size'] = test_df_imgs['image_filename'].apply(getDimensions)
 test_df_imgs['width'] = test_df_imgs['temp_size'].apply(lambda x : x[0])
 test_df_imgs['height'] = test_df_imgs['temp_size'].apply(lambda x : x[1])
 test_df_imgs = test_df_imgs.drop(['temp_size'], axis=1)
 aggs = {
'image_size': ['sum', 'mean', 'var'],
'width': ['sum', 'mean', 'var'],
'height': ['sum', 'mean', 'var'],
}
agg_train_imgs = train_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()
agg_test_imgs = test_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index()
agg_imgs = pd.concat([agg_train_imgs, agg_test_imgs], axis=0 ).reset_index(drop=True )
x_train = x_train/255.0
test = test/255.0<categorify>
X_temp = X_temp.merge(agg_imgs, how='left', on='PetID' )
y_train = to_categorical(y_train, num_classes = 10 )<split>
X_temp = X_temp.drop(to_drop_columns, axis=1 )
x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size = 0.1, random_state=2)
<choose_model_class>
X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1 )
model = Sequential()
model.add(Conv2D(64,kernel_size=5,padding = 'Same',activation='relu',input_shape=(28,28,1)))
 model.add(Conv2D(64,kernel_size=5,padding = 'Same',activation='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.40))
model.add(Conv2D(64,kernel_size=3,padding = 'Same',activation='relu'))
model.add(Conv2D(64,kernel_size=3,padding = 'Same',activation='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.40))
model.add(Flatten())
 model.add(Dense(128, activation = "relu"))
model.add(Dropout(0.40))
model.add(Dense(10, activation = "softmax"))
model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"] )<define_variables>
X_train_non_null.isnull().any().any() , X_test_non_null.isnull().any().any()
epochs = 30
batch_size = 100<init_hyperparams>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(x_train )<choose_model_class>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']
LR_reduction = ReduceLROnPlateau(monitor='val_accuracy',
patience = 2,
verbose = 1,
factor = 0.5,
min_lr = 0.00001 )<train_model>
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'eta': 0.0123,
'subsample': 0.8,
'colsample_bytree': 0.85,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
}
model.fit_generator(datagen.flow(x_train, y_train, batch_size = batch_size),
epochs = epochs, validation_data =(x_val,y_val), steps_per_epoch=x_train.shape[0] // batch_size, callbacks=[LR_reduction] )<save_to_csv>
def run_xgb(params, X_train, X_test):
n_splits = 10
verbose_eval = 1000
num_rounds = 60000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test
results = model.predict(test)
 results = np.argmax(results, axis = 1)
 results = pd.Series(results, name="Label")
 submission = pd.concat([pd.Series(range(1,28001), name = "ImageId"), results], axis = 1)
 submission.to_csv("submission.csv", index=False )<import_modules>
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null )
import os
import random
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.optim import Adam
from torch.utils.data import DataLoader
from torchvision import transforms
import numpy as np
import matplotlib.pyplot as plt
from torch.nn import CrossEntropyLoss
from tqdm import tqdm
from collections import Counter<load_from_csv>
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk )
root = "/kaggle/input/digit-recognizer"
train_data = np.loadtxt(os.path.join(root,"train.csv"),delimiter=",",skiprows=1)
 test_data = np.loadtxt(os.path.join(root,"test.csv"),delimiter=",",skiprows=1 )<import_modules>
coefficients_ = coefficients.copy()
coefficients_[0] = 1.66
coefficients_[1] = 2.13
coefficients_[3] = 2.85
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}' )
!nvidia-smi<categorify>
Counter(train_predictions )
class Dataset:

 def __init__(self,data,targets,transform=None):
self.data = data
self.targets = targets
self.transform = transform
def __len__(self):
return len(self.data)
 def __getitem__(self,idx):
if self.transform == None:
return self.data[idx],self.targets[idx]
else:
return self.transform(self.data[idx]),self.targets[idx]<prepare_x_and_y>
Counter(test_predictions )
transform = transforms.Compose([transforms.ToPILImage() ,
transforms.RandomResizedCrop(size=28,scale=(0.9,1.0),ratio=(0.9,1,15)) ,
transforms.RandomAffine(degrees=12,translate=(1/7,1/7),shear=12),
transforms.RandomRotation(degrees=12),
transforms.ToTensor() ])
 x_train = train_data[:,1:].reshape(-1,28,28 ).astype(np.uint8)
 y_train = torch.LongTensor(train_data[:,0])
 train_dataset = Dataset(x_train,y_train,transform )<feature_engineering>
submission1 = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions} )
class CNN(nn.Module):
def __init__(self):
super(CNN,self ).__init__()
self.layer1 = self.get_conv_block(1,64)
 self.layer2 = self.get_conv_block(64,128,paddings=(0,1))
self.layer3 = self.get_conv_block(128,256)
 self.fc1 = nn.Linear(256*3*3,2048)
 self.fc2 = nn.Linear(2048,512)
 self.fc3 = nn.Linear(512,10)
 def forward(self,x):
x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x ).flatten(start_dim=1)
 x = F.dropout(F.relu(self.fc1(x)) ,0.7,training=self.training)
 out = self.fc3(F.relu(self.fc2(x)))
 return out
def get_conv_block(self,in_chan,out_chan,strides=(1,1),paddings=(1,1)) :
return nn.Sequential(
 nn.Conv2d(in_chan,out_chan,3,strides[0],paddings[0]),
nn.ReLU() ,
nn.BatchNorm2d(out_chan),
nn.Conv2d(out_chan,out_chan,3,strides[0],paddings[1]),
nn.ReLU() ,
nn.BatchNorm2d(out_chan),
nn.MaxPool2d(2)
 )<train_model>
%matplotlib inline
np.random.seed(seed=1337)
 warnings.filterwarnings('ignore')
 split_char = '/'
os.listdir('.. /input' )
device = torch.device("cuda")if torch.cuda.is_available() else torch.device("cpu")
 random.seed(1234)
 np.random.seed(1234)
 torch.random.manual_seed(1234)
 epochs = 80
batch_size = 512
trainloader = DataLoader(train_dataset,batch_size,shuffle=True,pin_memory=True)
 def train(dataloader,net,optimizer,loss_fn,epochs=50):
for n in range(epochs):
with tqdm(dataloader,desc=f"{n+1}/{epochs} epochs")as t:
running_loss = 0.0
running_correct = 0
running_total = 0
for x,y in t:
out = net(x.to(device))
pred = out.max(dim=1)[1]
loss = loss_fn(out,y.to(device))
opt.zero_grad()
loss.backward()
opt.step()
running_loss += loss.item() *x.size(0)
 running_correct +=(pred==y.to(device)).sum().item()
running_total += x.size(0)
 t.set_postfix({"train_loss":running_loss/running_total,"train_acc":running_correct/running_total})
 models = []
for i in range(5):
print("Training {} model".format(str(i+1)+["st","nd","rd","th","th"][i]))
cnn = CNN()
cnn.to(device)
 opt = Adam(cnn.parameters() ,lr=1e-4)
 loss_fn = CrossEntropyLoss()
train(trainloader,cnn,opt,loss_fn,epochs)
 models.append(cnn )<categorify>
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv' )
cnn.eval()
train_dataset.transform = transforms.ToTensor()
trainloader = DataLoader(train_dataset,batch_size=512,shuffle=False,pin_memory=True)
 train_preds = []
for cnn in models:
train_pred = []
with torch.no_grad() :
for x,_ in trainloader:
out = cnn(x.to(device))
pred = out.max(dim=1)[1]
train_pred.append(pred.detach().cpu().numpy())
 train_preds.append(np.concatenate(train_pred))
train_preds = list(zip(*train_preds))
train_pred = np.array(list(map(lambda x: Counter(x ).most_common(1)[0][0],train_preds)))
 train_acc =(train_pred == y_train.numpy() ).astype("float" ).mean()
print("The training accurary is {}".format(train_acc))<predict_on_test>
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv')
 train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print(f'num of train images files: {len(train_image_files)}')
 print(f'num of train metadata files: {len(train_metadata_files)}')
 print(f'num of train sentiment files: {len(train_sentiment_files)}')
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print(f'num of test images files: {len(test_image_files)}')
 print(f'num of test metadata files: {len(test_metadata_files)}')
 print(f'num of test sentiment files: {len(test_sentiment_files)}')
x_test = test_data.reshape(-1,1,28,28)
 x_test = torch.Tensor(x_test)/255.
 test_preds = []
for cnn in models:
test_pred = []
with torch.no_grad() :
for i in range(0,len(x_test),batch_size):
out = cnn(x_test[i:i+batch_size].to(device))
pred = out.max(dim=1)[1]
test_pred.append(pred.detach().cpu().numpy())
 test_preds.append(np.concatenate(test_pred))
test_preds = list(zip(*test_preds))
test_pred = list(map(lambda x: Counter(x ).most_common(1)[0][0],test_preds))<save_to_csv>
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_json_file(self, filename):
with open(filename, 'r', encoding='utf-8')as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns')
 file_sentences_sentiment_df = pd.DataFrame(
 {
'magnitude_sum': file_sentences_sentiment['magnitude'].sum(axis=0),
'score_sum': file_sentences_sentiment['score'].sum(axis=0),
}, index=[0] 
)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
df_sentiment = pd.concat([df_sentiment, file_sentences_sentiment_df], axis=1)
 df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = f'.. /input/petfinder-adoption-prediction/{mode}_sentiment/{pet_id}.json'
try:
sentiment_file = pet_parser.open_json_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob(f'.. /input/petfinder-adoption-prediction/{mode}_metadata/{pet_id}*.json'))
if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_json_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()
imageid = pd.Series(np.arange(len(test_pred)))+1
df = pd.DataFrame({"ImageId":imageid,"Label":test_pred})
 df.set_index("ImageId")
 df.to_csv("/kaggle/working/test_pred.csv",index=False )<set_options>
aggregates = ['sum', 'mean']
sent_agg = ['sum']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 train_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 test_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()
%matplotlib inline 
<load_from_csv>
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]
train_data = pd.read_csv("/kaggle/input/digit-recognizer/train.csv")
 train_data.head()<load_from_csv>
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape)
test_data = pd.read_csv("/kaggle/input/digit-recognizer/test.csv")
 test_data.head()<import_modules>
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False)
 X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID']
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID')
 for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('none' )
from sklearn.model_selection import train_test_split<import_modules>
n_components = 5
text_features = []
for i in X_text.columns:
print(f'generating features from: {i}')
 tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_{}_'.format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )
from sklearn.model_selection import train_test_split<prepare_x_and_y>
X_temp = X_temp.drop(to_drop_columns, axis=1)
 X_train = X_temp.loc[np.isfinite(X_temp.AdoptionSpeed), :]
X_test = X_temp.loc[~np.isfinite(X_temp.AdoptionSpeed), :]
X_test = X_test.drop(['AdoptionSpeed'], axis=1)
 assert X_train.shape[0] == train.shape[0]
assert X_test.shape[0] == test.shape[0]
train_cols = X_train.columns.tolist()
train_cols.remove('AdoptionSpeed')
 test_cols = X_test.columns.tolist()
assert np.all(train_cols == test_cols)
 X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1 )
y = train_data['label'] 
<drop_column>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )
df_train = train_data.drop(['label'], axis=1 )<import_modules>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']
from sklearn import preprocessing
from sklearn.model_selection import train_test_split
import sklearn.metrics as metrics<feature_engineering>
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'silent': 1,
}
def run_xgb(params, X_train, X_test):
n_splits = 5
verbose_eval = 1000
num_rounds = 30000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null )
df_train = df_train/255
df_train[0:5]<categorify>
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk )
y = to_categorical(y, num_classes = 10)
 y.shape<split>
coefficients_ = coefficients.copy()
coefficients_[0] = 1.65
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_ ).astype(np.int8)
 submission2 = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions} )
X_train, X_test, y_train, y_test = train_test_split(df_train, y, test_size = 0.1, random_state=42, stratify=y )<define_search_model>
train_df = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 img_size = 256
batch_size = 16
input_shape =(28,28,1)
 X_input = Input(input_shape)
 x = Conv2D(64,(3,3),strides=(1,1),name='layer_conv1',padding='same' )(X_input)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 x = MaxPooling2D(( 2,2),name='maxPool1' )(x)
 x = Conv2D(32,(3,3),strides=(1,1),name='layer_conv2',padding='same' )(x)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 x = MaxPooling2D(( 2,2),name='maxPool2' )(x)
 x = Conv2D(32,(3,3),strides=(1,1),name='conv3',padding='same' )(x)
 x = BatchNormalization()(x)
 x = Activation('relu' )(x)
 x = MaxPooling2D(( 2,2), name='maxPool3' )(x)
 x = Flatten()(x)
 x = Dense(64,activation ='relu',name='fc0' )(x)
 x = Dropout(0.25 )(x)
 x = Dense(32,activation ='relu',name='fc1' )(x)
 x = Dropout(0.25 )(x)
 x = Dense(10,activation ='softmax',name='fc2' )(x)
 conv_model = Model(inputs=X_input, outputs=x, name='Predict')
 conv_model.summary()<train_model>
pet_ids = train_df['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
conv_model.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy'])
 conv_model.fit(X_train, y_train, epochs=10, batch_size=100, validation_data=(X_test,y_test))<train_model>
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights = ".. /input/densenet121weights/densenet121_weights_tf_dim_ordering_tf_kernels_notop.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )
sgd = SGD(lr=0.0005, momentum=0.5, decay=0.0, nesterov=False)
 conv_model.compile(optimizer=sgd,loss='categorical_crossentropy',metrics=['accuracy'])
 conv_model.fit(X_train, y_train, epochs=30, validation_data=(X_test, y_test))<init_hyperparams>
features = {}
for b in tqdm_notebook(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<train_model>
train_feats = pd.DataFrame.from_dict(features, orient='index' )
hist = conv_model.fit_generator(datagen.flow(X_train,y_train),
epochs = 1, validation_data =(X_test,y_test),
verbose = 2)
<train_model>
test_df = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 pet_ids = test_df['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm_notebook(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/test_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
test_feats = pd.DataFrame.from_dict(features, orient='index' )
data_generator = ImageDataGenerator(rescale=1./255,
rotation_range=1,
zoom_range=0.1,
width_shift_range=0.05,
height_shift_range=0.05)
 data_generator.fit(X_train)
<feature_engineering>
train_feats.reset_index(inplace=True)
 test_feats.reset_index(inplace=True )
df_test = test_data/255
df_test[0:5]<save_to_csv>
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv')
 labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv')
 train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
train_img_features = train_feats
test_img_features = test_feats
col_names =["PetID"] + ["{}_img_feature".format(_)for _ in range(256)]
train_img_features.columns = col_names
test_img_features.columns = col_names
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_metadata_file(self, filename):

 with open(filename, 'r')as f:
metadata_file = json.load(f)
 return metadata_file
def open_sentiment_file(self, filename):

 with open(filename, 'r')as f:
sentiment_file = json.load(f)
 return sentiment_file
def open_image_file(self, filename):

 image = np.asarray(Image.open(filename))
return image
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 if self.extract_sentiment_text:
file_sentences_text = [x['text']['content'] for x in file['sentences']]
file_sentences_text = self.sentence_sep.join(file_sentences_text)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns' ).sum()
file_sentences_sentiment = file_sentences_sentiment.add_prefix('document_' ).to_dict()
file_sentiment.update(file_sentences_sentiment)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
if self.extract_sentiment_text:
df_sentiment['text'] = file_sentences_text
df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations'][:int(len(file['labelAnnotations'])* 0.3)]
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = '.. /input/petfinder-adoption-prediction/{}_sentiment/{}.json'.format(mode, pet_id)
 try:
sentiment_file = pet_parser.open_sentiment_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob('.. /input/petfinder-adoption-prediction/{}_metadata/{}*.json'.format(mode, pet_id)))
 if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_metadata_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
def agg_features(df_metadata, df_sentiment):
aggregates = ['mean', "median", 'sum', "var", "std", "min", "max", "nunique"]
metadata_desc = df_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
metadata_desc = metadata_desc.reset_index()
metadata_desc['metadata_annots_top_desc'] = metadata_desc['metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
metadata_gr = df_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in metadata_gr.columns:
if 'PetID' not in i:
metadata_gr[i] = metadata_gr[i].astype(float)
 metadata_gr = metadata_gr.groupby(['PetID'] ).agg(aggregates)
 metadata_gr.columns = pd.Index(['{}_{}_{}'.format(prefix, c[0], c[1].upper())for c in metadata_gr.columns.tolist() ])
 metadata_gr = metadata_gr.reset_index()
sentiment_desc = df_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
sentiment_desc = sentiment_desc.reset_index()
sentiment_desc['sentiment_entities'] = sentiment_desc['sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
sentiment_gr = df_sentiment.drop(['sentiment_entities'], axis=1)
 for i in sentiment_gr.columns:
if 'PetID' not in i:
sentiment_gr[i] = sentiment_gr[i].astype(float)
 sentiment_gr = sentiment_gr.groupby(['PetID'] ).agg(aggregates)
 sentiment_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in sentiment_gr.columns.tolist() ])
 sentiment_gr = sentiment_gr.reset_index()
return sentiment_gr, metadata_gr, metadata_desc, sentiment_desc
def breed_features(df, _labels_breed):
breed_main = df[['Breed1']].merge(_labels_breed, how='left', left_on='Breed1', right_on='BreedID', suffixes=('', '_main_breed'))
breed_main = breed_main.iloc[:, 2:]
breed_main = breed_main.add_prefix('main_breed_')
 breed_second = df[['Breed2']].merge(_labels_breed, how='left', left_on='Breed2', right_on='BreedID', suffixes=('', '_second_breed'))
breed_second = breed_second.iloc[:, 2:]
breed_second = breed_second.add_prefix('second_breed_')
 return breed_main, breed_second
def impact_coding(data, feature, target='y'):

 n_folds = 20
n_inner_folds = 10
impact_coded = pd.Series()
oof_default_mean = data[target].mean()
kf = KFold(n_splits=n_folds, shuffle=True)
 oof_mean_cv = pd.DataFrame()
split = 0
for infold, oof in kf.split(data[feature]):
impact_coded_cv = pd.Series()
kf_inner = KFold(n_splits=n_inner_folds, shuffle=True)
 inner_split = 0
inner_oof_mean_cv = pd.DataFrame()
oof_default_inner_mean = data.iloc[infold][target].mean()
for infold_inner, oof_inner in kf_inner.split(data.iloc[infold]):
oof_mean = data.iloc[infold_inner].groupby(by=feature)[target].mean()
impact_coded_cv = impact_coded_cv.append(data.iloc[infold].apply(
 lambda x: oof_mean[x[feature]]
if x[feature] in oof_mean.index
else oof_default_inner_mean
, axis=1))
inner_oof_mean_cv = inner_oof_mean_cv.join(pd.DataFrame(oof_mean), rsuffix=inner_split, how='outer')
 inner_oof_mean_cv.fillna(value=oof_default_inner_mean, inplace=True)
 inner_split += 1
oof_mean_cv = oof_mean_cv.join(pd.DataFrame(inner_oof_mean_cv), rsuffix=split, how='outer')
 oof_mean_cv.fillna(value=oof_default_mean, inplace=True)
 split += 1
impact_coded = impact_coded.append(data.iloc[oof].apply(
 lambda x: inner_oof_mean_cv.loc[x[feature]].mean()
if x[feature] in inner_oof_mean_cv.index
else oof_default_mean
, axis=1))
return impact_coded, oof_mean_cv.mean(axis=1), oof_default_mean
def frequency_encoding(df, col_name):
new_name = "{}_counts".format(col_name)
 new_col_name = "{}_freq".format(col_name)
 grouped = df.groupby(col_name ).size().reset_index(name=new_name)
 df = df.merge(grouped, how = "left", on = col_name)
 df[new_col_name] = df[new_name]/df[new_name].count()
del df[new_name]
return df
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator)
 class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))
def train_lightgbm(X_train, X_test, params, n_splits, num_rounds, verbose_eval, early_stop):
kfold = StratifiedKFold(n_splits=n_splits, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_index, valid_index in kfold.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_index, :]
X_val = X_train.iloc[valid_index, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 print('
y_tr distribution: {}'.format(Counter(y_tr)))
 d_train = lgb.Dataset(X_tr, label=y_tr)
 d_valid = lgb.Dataset(X_val, label=y_val)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 val_pred = model.predict(X_val, num_iteration=model.best_iteration)
 test_pred = model.predict(X_test, num_iteration=model.best_iteration)
 oof_train[valid_index] = val_pred
oof_test[:, i] = test_pred
i += 1
return oof_train, oof_test
pet_parser = PetFinderParser()
def main() :
train_pet_ids = train.PetID.unique()
test_pet_ids = test.PetID.unique()
dfs_train = Parallel(n_jobs=6, verbose=1 )(
 delayed(extract_additional_features )(i, mode='train')for i in train_pet_ids)
 train_dfs_sentiment = [x[0] for x in dfs_train if isinstance(x[0], pd.DataFrame)]
train_dfs_metadata = [x[1] for x in dfs_train if isinstance(x[1], pd.DataFrame)]
train_dfs_sentiment = pd.concat(train_dfs_sentiment, ignore_index=True, sort=False)
 train_dfs_metadata = pd.concat(train_dfs_metadata, ignore_index=True, sort=False)
 dfs_test = Parallel(n_jobs=6, verbose=1 )(
 delayed(extract_additional_features )(i, mode='test')for i in test_pet_ids)
 test_dfs_sentiment = [x[0] for x in dfs_test if isinstance(x[0], pd.DataFrame)]
test_dfs_metadata = [x[1] for x in dfs_test if isinstance(x[1], pd.DataFrame)]
test_dfs_sentiment = pd.concat(test_dfs_sentiment, ignore_index=True, sort=False)
 test_dfs_metadata = pd.concat(test_dfs_metadata, ignore_index=True, sort=False)
 train_sentiment_gr, train_metadata_gr, train_metadata_desc, train_sentiment_desc = agg_features(train_dfs_metadata, train_dfs_sentiment)
 test_sentiment_gr, test_metadata_gr, test_metadata_desc, test_sentiment_desc = agg_features(test_dfs_metadata, test_dfs_sentiment)
 train_proc = train.copy()
for tr in [train_sentiment_gr, train_metadata_gr, train_metadata_desc, train_sentiment_desc]:
train_proc = train_proc.merge(tr, how='left', on='PetID')
 test_proc = test.copy()
for ts in [test_sentiment_gr, test_metadata_gr, test_metadata_desc, test_sentiment_desc]:
test_proc = test_proc.merge(
 ts, how='left', on='PetID')
 train_proc = pd.merge(train_proc, train_img_features, on="PetID")
 test_proc = pd.merge(test_proc, test_img_features, on="PetID")
 train_breed_main, train_breed_second = breed_features(train_proc, labels_breed)
 train_proc = pd.concat([train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main, test_breed_second = breed_features(test_proc, labels_breed)
 test_proc = pd.concat([test_proc, test_breed_main, test_breed_second], axis=1)
 X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False)
 column_types = X.dtypes
int_cols = column_types[column_types == 'int']
float_cols = column_types[column_types == 'float']
cat_cols = column_types[column_types == 'object']
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID']
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID')
 for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('<MISSING>')
 n_components = 5
text_features = []
for i in X_text.columns:
print('generating features from: {}'.format(i))
svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 nmf_ = NMF(
 n_components=n_components, random_state=1337)
 tfidf_col = TfidfVectorizer().fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('SVD_{}_'.format(i))
nmf_col = nmf_.fit_transform(tfidf_col)
 nmf_col = pd.DataFrame(nmf_col)
 nmf_col = nmf_col.add_prefix('NMF_{}_'.format(i))
text_features.append(svd_col)
 text_features.append(nmf_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1)
 X_temp["name_length"] = X_temp.Name[X_temp.Name.isnull() ].map(lambda x: len(str(x)))
 X_temp["name_length"] = X_temp.Name.map(lambda x: len(str(x)))
 X_temp = X_temp.drop(to_drop_columns, axis=1)
 X_train = X_temp.loc[np.isfinite(X_temp.AdoptionSpeed), :]
X_test = X_temp.loc[~np.isfinite(X_temp.AdoptionSpeed), :]
X_test = X_test.drop(['AdoptionSpeed'], axis=1)
 print('X_train shape: {}'.format(X_train.shape))
print('X_test shape: {}'.format(X_test.shape))
assert X_train.shape[0] == train.shape[0]
assert X_test.shape[0] == test.shape[0]
train_cols = X_train.columns.tolist()
train_cols.remove('AdoptionSpeed')
 test_cols = X_test.columns.tolist()
np.random.seed(13)
 categorical_features = ["Type", "Breed1", "Breed2", "Color1" ,"Color2", "Color3", "State"]
impact_coding_map = {}
for f in categorical_features:
print("Impact coding for {}".format(f))
X_train["impact_encoded_{}".format(f)], impact_coding_mapping, default_coding = impact_coding(X_train, f, target="AdoptionSpeed")
 impact_coding_map[f] =(impact_coding_mapping, default_coding)
 mapping, default_mean = impact_coding_map[f]
X_test["impact_encoded_{}".format(f)] = X_test.apply(lambda x: mapping[x[f]] if x[f] in mapping
else default_mean, axis=1)
 for cat in categorical_features:
X_train = frequency_encoding(X_train, cat)
 X_test = frequency_encoding(X_test, cat)
 params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 70,
'max_depth': 9,
'learning_rate': 0.01,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.02,
'min_child_samples': 150,
'min_child_weight': 0.02,
'lambda_l2': 0.0475,
'verbosity': -1,
'data_random_seed': 17}
early_stop = 500
verbose_eval = 100
num_rounds = 10000
n_splits = 5
oof_train, oof_test = train_lightgbm(X_train, X_test, params, n_splits, num_rounds, verbose_eval, early_stop)
 optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(oof_train, coefficients)
 print("
Valid Counts = ", Counter(X_train['AdoptionSpeed'].values))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, pred_test_y_k)
 print("QWK = ", qwk)
 coefficients_ = coefficients.copy()
coefficients_[0] = 1.645
coefficients_[1] = 2.115
coefficients_[3] = 2.84
train_predictions = optR.predict(oof_train, coefficients_ ).astype(int)
 print('train pred distribution: {}'.format(Counter(train_predictions)))
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_)
 print('test pred distribution: {}'.format(Counter(test_predictions)))
 submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions.astype(np.int32)})
 return submission
submission3 = main()
<save_to_csv><EOS> 
submission1["AdoptionSpeed"] = submission1.AdoptionSpeed*0.4 + submission2.AdoptionSpeed*0.2 + submission3.AdoptionSpeed*0.4
submission1["AdoptionSpeed"] = submission1["AdoptionSpeed"].round().astype(int)
 submission1.to_csv('submission.csv', index=False )