Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<merge>
%matplotlib inline
np.random.seed(seed=1337)
 warnings.filterwarnings('ignore')
 split_char = '/'
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]<merge>
class Config:
num_seed_stratified_group = 4
num_seed_stratified = 4
num_seed_group = 4
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )<concatenate>
train = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 test = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 sample_submission = pd.read_csv('.. /input/petfinder-adoption-prediction/test/sample_submission.csv' )
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False )<define_variables>
import cv2
from PIL import Image as IMG
import os
from keras.applications.densenet import preprocess_input, DenseNet121
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID']<merge>
img_size = 256
batch_size = 32
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID' )<feature_engineering>
pet_ids = train['PetID'].values
im_feature_dict = pd.DataFrame(index=pet_ids, columns=['light_percent', 'dark_percent', 'sharpness',
'top1_size', 'top1_width', 'top1_height'] ).add_prefix('im_')
 for pid in tqdm_notebook(pet_ids):
try:
im, original_shape = load_raw_image(".. /input/petfinder-adoption-prediction/train_images/", pid)
 im_feature_dict.loc[pid, 'im_top1_size'] = original_shape[0] * original_shape[1]
im_feature_dict.loc[pid, 'im_top1_width'] = original_shape[1]
im_feature_dict.loc[pid, 'im_top1_height'] = original_shape[0]
im_feature_dict.loc[pid, 'im_sharpness'] = get_blurrness_score(im)
 im = IMG.fromarray(np.uint8(im))
light_percent, dark_percent = color_analysis(im)
 im_feature_dict.loc[pid, 'im_light_percent'] = light_percent
im_feature_dict.loc[pid, 'im_dark_percent'] = dark_percent
except AttributeError:
pass
pet_ids = test['PetID'].values
test_im_feature_dict = pd.DataFrame(index=pet_ids, columns=['light_percent', 'dark_percent', 'sharpness',
'top1_size', 'top1_width', 'top1_height'] ).add_prefix('im_')
 for pid in tqdm_notebook(pet_ids):
try:
im, original_shape = load_raw_image(".. /input/petfinder-adoption-prediction/test_images/", pid)
 test_im_feature_dict.loc[pid, 'im_top1_size'] = original_shape[0] * original_shape[1]
test_im_feature_dict.loc[pid, 'im_top1_width'] = original_shape[1]
test_im_feature_dict.loc[pid, 'im_top1_height'] = original_shape[0]
test_im_feature_dict.loc[pid, 'im_sharpness'] = get_blurrness_score(im)
 im = IMG.fromarray(np.uint8(im))
light_percent, dark_percent = color_analysis(im)
 test_im_feature_dict.loc[pid, 'im_light_percent'] = light_percent
test_im_feature_dict.loc[pid, 'im_dark_percent'] = dark_percent
except AttributeError:
pass
for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]<merge>
train = train.merge(im_feature_dict, how='left', left_on='PetID', right_index=True)
 print(train.shape)
 train.head()
X_temp = X_temp.merge(img_features, how='left', on='PetID' )<define_search_space>
test = test.merge(test_im_feature_dict, how='left', left_on='PetID', right_index=True)
 print(test.shape)
 test.head()
len(np.asarray(np.array([[1,2,3],[4,5,6]])).shape )<define_variables>
img_size = 256
batch_size = 32
replace_backslash = lambda x: x.replace('\', '/')
 vectorized_replace_backslash = np.vectorize(replace_backslash)
 aggs = {
'image_blurness': ['sum', 'mean', 'var'],
'image_size': ['sum', 'mean', 'var'],
'image_width': ['sum', 'mean', 'var'],
'image_height': ['sum', 'mean', 'var'],
}<feature_engineering>
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights=".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )
def get_image_features(img_info):
img_info["image_size"] = getSize(img_info['image_filename'])
 try:
im = Image.open(img_info['image_filename'])
 except Exception as e:
im = np.NaN
if im != np.NaN:
dimension = getDimensions(im)
 img_info["image_width"] = dimension[0]
img_info["image_height"] = dimension[1]
img_info["image_blurness"] = get_blurrness_score(im)
 else:
img_info["image_width"] = np.NaN
img_info["image_height"] = np.NaN
img_info["image_blurness"] = np.NaN
return img_info<feature_engineering>
pet_ids = train['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
train_df_ids = train[['PetID']]
train_image_files = vectorized_replace_backslash(train_image_files)
 train_df_imgs = pd.DataFrame(train_image_files)
 train_df_imgs.columns = ['image_filename']
train_imgs_pets = train_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 train_image_features = train_df_imgs.assign(PetID=train_imgs_pets)
 with timer("getting image extra image features for train: "):
train_image_features = train_image_features.apply(get_image_features, axis=1 )<groupby>
train_feats = pd.DataFrame.from_dict(features, orient='index')
 train_feats.columns = [f'pic_{i}' for i in range(train_feats.shape[1])]
agg_train_imgs = train_image_features.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()<feature_engineering>
pet_ids = test['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/test_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
test_df_ids = test[['PetID']]
test_image_files = vectorized_replace_backslash(test_image_files)
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 test_image_features = test_df_imgs.assign(PetID=test_imgs_pets)
 with timer("getting image extra image features for test: "):
test_image_features = test_image_features.apply(get_image_features, axis=1 )<groupby>
test_feats = pd.DataFrame.from_dict(features, orient='index')
 test_feats.columns = [f'pic_{i}' for i in range(test_feats.shape[1])]
agg_test_imgs = test_image_features.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index()<concatenate>
train_feats = train_feats.reset_index()
train_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 test_feats = test_feats.reset_index()
test_feats.rename({'index': 'PetID'}, axis='columns', inplace=True )
agg_imgs = pd.concat([agg_train_imgs, agg_test_imgs], axis=0 ).reset_index(drop=True)
 agg_imgs_columns = list(agg_imgs.columns)
 agg_imgs_columns.remove('PetID' )<merge>
all_ids = pd.concat([train, test], axis=0, ignore_index=True, sort=False)[['PetID']]
all_ids.shape
X_temp = X_temp.merge(agg_imgs, how='left', on='PetID' )<feature_engineering>
n_components = 32
svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 features_df = pd.concat([train_feats, test_feats], axis=0)
 features = features_df[[f'pic_{i}' for i in range(256)]].values
svd_col = svd_.fit_transform(features)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('IMG_SVD_')
 img_features = pd.concat([all_ids, svd_col], axis=1 )
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('none' )<feature_engineering>
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )
n_components = 32
text_features = []
for i in X_text.columns:
print(f'generating features from: {i}')
 tfv = TfidfVectorizer(max_features=None, strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_{}_'.format(i))
current_text = X_temp[i].str.lower().astype(str)
 X_temp[i] = X_temp[i].astype(str)
 X_temp[i + '_num_chars'] = current_text.apply(len)
 X_temp[i + '_num_words'] = current_text.apply(lambda comment: len(comment.split()))
X_temp[i + '_num_unique_words'] = current_text.apply(lambda comment: len(set(w for w in comment.split())))
 X_temp[i + '_words_vs_unique'] = X_temp[i + '_num_unique_words'] / X_temp[i+'_num_words'] * 100
X_temp[i + '_count_letters'] = current_text.apply(lambda x: len(str(x)))
 X_temp[i + "_count_punctuations"] = current_text.apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
X_temp[i + "_mean_word_len"] = current_text.apply(lambda x: np.mean([len(w)for w in str(x ).split() ]))
text_features.append(svd_col)
 X_temp['words_vs_unique_description'] = X_temp['Description_num_unique_words'] / X_temp['Description_num_words'] * 100
X_temp['words_vs_unique_description'] = X_temp['Description_num_unique_words'] / X_temp['Description_num_words'] * 100
X_temp['len_description'] = X_temp['Description'].apply(lambda x: len(x))
X_temp['average_description_word_length'] = X_temp['len_description'] / X_temp['Description_num_words']<feature_engineering>
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print(f'num of train images files: {len(train_image_files)}')
 print(f'num of train metadata files: {len(train_metadata_files)}')
 print(f'num of train sentiment files: {len(train_sentiment_files)}')
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print(f'num of test images files: {len(test_image_files)}')
 print(f'num of test metadata files: {len(test_metadata_files)}')
 print(f'num of test sentiment files: {len(test_sentiment_files)}' )
embeddings_index = {}
time_start_load_embedding = time.time()
with open('.. /input/fatsttext-common-crawl/crawl-300d-2M/crawl-300d-2M.vec', encoding='utf8')as f:
for line in f:
values = line.rstrip().rsplit(' ')
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
print("finished loading embeddings file in: ", time.time() - time_start_load_embedding )<feature_engineering>
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_json_file(self, filename):
with open(filename, 'r', encoding='utf-8')as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns')
 file_sentences_sentiment_df = pd.DataFrame(
 {
'magnitude_sum': file_sentences_sentiment['magnitude'].sum(axis=0),
'magnitude_mean': file_sentences_sentiment['magnitude'].mean(axis=0),
'magnitude_var': file_sentences_sentiment['magnitude'].var(axis=0),
'magnitude_max': file_sentences_sentiment['magnitude'].max(axis=0),
'magnitude_min': file_sentences_sentiment['magnitude'].min(axis=0),
'score_sum': file_sentences_sentiment['score'].sum(axis=0),
'score_mean': file_sentences_sentiment['score'].mean(axis=0),
'score_var': file_sentences_sentiment['score'].var(axis=0),
'score_max': file_sentences_sentiment['score'].max(axis=0),
'score_min': file_sentences_sentiment['score'].min(axis=0),
}, index=[0] 
)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
df_sentiment = pd.concat([df_sentiment, file_sentences_sentiment_df], axis=1)
 df_sentiment['entities'] = file_entities
df_sentiment['language'] = file['language']
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_score_var = np.asarray([x['score'] for x in file_annots] ).var()
file_top_score_max = np.asarray([x['score'] for x in file_annots] ).max()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_score_var = np.nan
file_top_score_max = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_score_var = np.asarray([x['score'] for x in file_colors] ).var()
file_color_score_max = np.asarray([x['score'] for x in file_colors] ).max()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_color_pixelfrac_var = np.asarray([x['pixelFraction'] for x in file_colors] ).var()
file_color_pixelfrac_max = np.asarray([x['pixelFraction'] for x in file_colors] ).max()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'annots_score_var': file_top_score_var,
'annots_score_max': file_top_score_max,
'color_score': file_color_score,
'color_score_var': file_color_score_var,
'color_score_max': file_color_score_max,
'color_pixelfrac': file_color_pixelfrac,
'color_pixelfrac_var': file_color_pixelfrac_var,
'color_pixelfrac_max': file_color_pixelfrac_max,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = f'.. /input/petfinder-adoption-prediction/{mode}_sentiment/{pet_id}.json'
try:
sentiment_file = pet_parser.open_json_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob(f'.. /input/petfinder-adoption-prediction/{mode}_metadata/{pet_id}*.json'))
if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_json_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()
def sum_embeddings_vector(x):
sum_bag_of_words = np.zeros(300)
 words = x.split(' ')
 count = 0
for w in words:
if w in embeddings_index:
sum_bag_of_words += embeddings_index.get(w)
 count += 1
if count > 0:
sum_bag_of_words /= count
return pd.Series(sum_bag_of_words )<feature_engineering>
train_dfs_sentiment.sentiment_language.value_counts()
svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 for i in X_text.columns:
emb_feat = X_text[i].apply(sum_embeddings_vector)
 emb_feat = svd_.fit_transform(emb_feat)
 emb_feat = pd.DataFrame(emb_feat)
 emb_feat = emb_feat.add_prefix('word_embedding_' + i + '_')
 text_features.append(emb_feat )<concatenate>
train_dfs_sentiment.groupby(['PetID'])['sentiment_language'].apply(lambda x: x.mode() ).reset_index().drop(columns='level_1' )
text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1 )<drop_column>
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_dfs_lang, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_dfs_lang, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]
for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )<define_variables>
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )
rescuer_ids = X_temp['RescuerID'].values<drop_column>
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False )
X_temp = X_temp.drop(to_drop_columns, axis=1 )<define_variables>
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName', 'sentiment_language']
to_drop_columns = ['PetID', 'Name', 'RescuerID']
X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1 )<count_missing_values>
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID' )
X_train_non_null.isnull().any().any() , X_test_non_null.isnull().any().any()<import_modules>
for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<train_on_grid>
X['sentiment_language'].value_counts()
def to_bins(x, borders):
for i in range(len(borders)) :
if x <= borders[i]:
return i
return len(borders)
 class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _loss(self, coef, X, y, idx):
X_p = np.array([to_bins(pred, coef)for pred in X])
 ll = -quadratic_weighted_kappa(y, X_p)
 return ll
def fit(self, X, y):
coef = [1.5, 2.0, 2.5, 3.0]
golden1 = 0.618
golden2 = 1 - golden1
ab_start = [(1, 2),(1.5, 2.5),(2, 3),(2.5, 3.5)]
for it1 in range(10):
for idx in range(4):
a, b = ab_start[idx]
coef[idx] = a
la = self._loss(coef, X, y, idx)
 coef[idx] = b
lb = self._loss(coef, X, y, idx)
 for it in range(20):
if la > lb:
a = b -(b - a)* golden1
coef[idx] = a
la = self._loss(coef, X, y, idx)
 else:
b = b -(b - a)* golden2
coef[idx] = b
lb = self._loss(coef, X, y, idx)
 self.coef_ = {'x': coef}
def predict(self, X, coef):
X_p = np.array([to_bins(pred, coef)for pred in X])
 return X_p
def coefficients(self):
return self.coef_['x']<import_modules>
X_temp['sentiment_language'].value_counts()
import lightgbm as lgb
import xgboost as xgb
from sklearn.model_selection import StratifiedKFold, GroupKFold<define_variables>
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('none' )
ntrain = X_train_non_null.shape[0]
ntest = X_test_non_null.shape[0]
NFOLDS = 10<groupby>
X_temp['Length_Description'] = X_text['Description'].map(len)
 X_temp['Length_metadata_annots_top_desc'] = X_text['metadata_annots_top_desc'].map(len)
 X_temp['Lengths_sentiment_entities'] = X_text['sentiment_entities'].map(len )
def stratified_group_k_fold(X, y, groups, k, seed=None):
labels_num = np.max(y)+ 1
y_counts_per_group = defaultdict(lambda: np.zeros(labels_num))
y_distr = Counter()
for label, g in zip(y, groups):
y_counts_per_group[g][label] += 1
y_distr[label] += 1
y_counts_per_fold = defaultdict(lambda: np.zeros(labels_num))
groups_per_fold = defaultdict(set)
 def eval_y_counts_per_fold(y_counts, fold):
y_counts_per_fold[fold] += y_counts
std_per_label = []
for label in range(labels_num):
label_std = np.std([y_counts_per_fold[i][label] / y_distr[label] for i in range(k)])
 std_per_label.append(label_std)
 y_counts_per_fold[fold] -= y_counts
return np.mean(std_per_label)
 groups_and_y_counts = list(y_counts_per_group.items())
 random.Random(seed ).shuffle(groups_and_y_counts)
 for g, y_counts in sorted(groups_and_y_counts, key=lambda x: -np.std(x[1])) :
best_fold = None
min_eval = None
for i in range(k):
fold_eval = eval_y_counts_per_fold(y_counts, i)
 if min_eval is None or fold_eval < min_eval:
min_eval = fold_eval
best_fold = i
y_counts_per_fold[best_fold] += y_counts
groups_per_fold[best_fold].add(g)
 all_groups = set(groups)
 for i in range(k):
train_groups = all_groups - groups_per_fold[i]
test_groups = groups_per_fold[i]
train_indices = [i for i, g in enumerate(groups)if g in train_groups]
test_indices = [i for i, g in enumerate(groups)if g in test_groups]
yield train_indices, test_indices<train_on_grid>
n_components = 16
text_features = []
for i in X_text.columns:
print(f'generating features from: {i}')
 tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_{}_'.format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )
def get_oof(clf, X, y, X_test, groups):
oof_train = np.zeros(( ntrain,))
oof_test = np.zeros(( ntest,))
oof_test_skf = np.empty(( NFOLDS, ntest))
for i,(train_index, test_index)in enumerate(stratified_group_k_fold(X, y, rescuer_ids, NFOLDS, 1337)) :
print('Training for fold: ', i + 1)
 x_tr = X.iloc[train_index, :]
y_tr = y[train_index]
x_te = X.iloc[test_index, :]
y_te = y[test_index]
clf.train(x_tr, y_tr, x_val=x_te, y_val=y_te)
 oof_train[test_index] = clf.predict(x_te)
 oof_test_skf[i, :] = clf.predict(X_test)
 oof_test[:] = oof_test_skf.mean(axis=0)
 return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1 )<train_model>
X_temp = X_temp.merge(img_features, how='left', on='PetID' )
class SklearnWrapper(object):
def __init__(self, clf, seed=0, params=None):
params['random_state'] = seed
self.clf = clf(**params)
 def train(self, x_train, y_train, **kwargs):
self.clf.fit(x_train, y_train)
 def predict(self, x):
return self.clf.predict(x)
 class XgbWrapper(object):
def __init__(self, params=None):
self.param = params
self.nrounds = params.pop('nrounds', 60000)
 self.early_stop_rounds = params.pop('early_stop_rounds', 2000)
 def train(self, x_train, y_train, **kwargs):
dtrain = xgb.DMatrix(x_train, label=y_train)
 dvalid = xgb.DMatrix(data=kwargs['x_val'], label=kwargs['y_val'])
 watchlist = [(dtrain, 'train'),(dvalid, 'valid')]
self.model = xgb.train(dtrain=dtrain, num_boost_round=self.nrounds, evals=watchlist, early_stopping_rounds=self.early_stop_rounds,
verbose_eval=1000, params=self.param)
 def predict(self, x):
return self.model.predict(xgb.DMatrix(x), ntree_limit=self.model.best_ntree_limit)
 class LGBWrapper(object):
def __init__(self, params=None):
self.param = params
self.num_rounds = params.pop('nrounds', 60000)
 self.early_stop_rounds = params.pop('early_stop_rounds', 2000)
 def train(self, x_train, y_train, **kwargs):
dtrain = lgb.Dataset(x_train, label=y_train)
 dvalid = lgb.Dataset(kwargs['x_val'], label=kwargs['y_val'])
 watchlist = [dtrain, dvalid]
print('training LightGBM with params: ', self.param)
 self.model = lgb.train(
 self.param,
train_set=dtrain,
num_boost_round=self.num_rounds,
valid_sets=watchlist,
verbose_eval=1000,
early_stopping_rounds=self.early_stop_rounds 
)
 def predict(self, x):
return self.model.predict(x, num_iteration=self.model.best_iteration )<choose_model_class>
train_df_ids = train[['PetID']]
test_df_ids = test[['PetID']]
debut = False
if debug:
train_pet_ids = train_pet_ids[:1000]
test_pet_ids = test_pet_ids[:500]
train_df_imgs = pd.DataFrame(train_image_files)
 train_df_imgs.columns = ['image_filename']
train_imgs_pets = train_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 def getSize(filename):
st = os.stat(filename)
 return st.st_size
def getDimensions(filename):
img_size = Image.open(filename ).size
return img_size
train_df_imgs['image_size'] = train_df_imgs['image_filename'].apply(getSize)
 train_df_imgs['temp_size'] = train_df_imgs['image_filename'].apply(getDimensions)
 train_df_imgs['width'] = train_df_imgs['temp_size'].apply(lambda x : x[0])
 train_df_imgs['height'] = train_df_imgs['temp_size'].apply(lambda x : x[1])
 train_df_imgs = train_df_imgs.drop(['temp_size'], axis=1)
 test_df_imgs['image_size'] = test_df_imgs['image_filename'].apply(getSize)
 test_df_imgs['temp_size'] = test_df_imgs['image_filename'].apply(getDimensions)
 test_df_imgs['width'] = test_df_imgs['temp_size'].apply(lambda x : x[0])
 test_df_imgs['height'] = test_df_imgs['temp_size'].apply(lambda x : x[1])
 test_df_imgs = test_df_imgs.drop(['temp_size'], axis=1)
 aggs = {
'image_size': ['sum', 'mean', 'var', 'max'],
'width': ['sum', 'mean', 'var', 'max'],
'height': ['sum', 'mean', 'var', 'max'],
}
agg_train_imgs = train_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()
agg_test_imgs = test_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index()
agg_imgs = pd.concat([agg_train_imgs, agg_test_imgs], axis=0 ).reset_index(drop=True )
lgbm_params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': 'rmse',
'nrounds': 50000,
'early_stop_rounds': 2000,
'max_depth': 4,
'num_leaves': 46,
'feature_fraction': 0.6,
'bagging_fraction': 0.9,
'bagging_freq': 8,
'learning_rate': 0.019,
'verbose': 0
}
lgb_wrapper = LGBWrapper(lgbm_params)
 with timer('Training LightGBM'):
lgb_oof_train, lgb_oof_test = get_oof(lgb_wrapper, X_train_non_null.drop(['AdoptionSpeed'], axis=1), X_train_non_null['AdoptionSpeed'].values.astype(int), X_test_non_null, groups=rescuer_ids )<train_model>
X_temp = X_temp.merge(agg_imgs, how='left', on='PetID' )
rf_params = {
'n_jobs': -1,
'n_estimators': 150,
'max_features': 'auto',
'max_depth': 20,
'min_samples_leaf': 2,
}
rf = SklearnWrapper(clf=RandomForestRegressor, seed=1337, params=rf_params)
 with timer('Training Random Forest'):
rf_oof_train, rf_oof_test = get_oof(rf, X_train_non_null.drop(['AdoptionSpeed'], axis=1), X_train_non_null['AdoptionSpeed'].values.astype(int), X_test_non_null, groups=rescuer_ids )<categorify>
X_temp = X_temp.drop(to_drop_columns, axis=1 )
et_params = {
'n_jobs': -1,
'n_estimators': 150,
'max_features': 'auto',
'max_depth': 50,
'min_samples_leaf': 2,
}
et = SklearnWrapper(clf=ExtraTreesRegressor, seed=1337, params=et_params)
 with timer('Training Extra trees'):
et_oof_train, et_oof_test = get_oof(et, X_train_non_null.drop(['AdoptionSpeed'], axis=1), X_train_non_null['AdoptionSpeed'].values.astype(int), X_test_non_null, groups=rescuer_ids )<categorify>
X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1 )
xgb_params = {
'eval_metric': 'rmse',
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
'seed': 1337,
'nrounds': 60000,
'early_stop_rounds': 2000,
'eta': 0.025,
'subsample': 0.8,
'colsample_bytree': 0.6000000000000001,
'gamma': 0.65,
'max_depth': 4,
'min_child_weight': 5.0,
'n_estimators': 1000,
}
xgbWrapper = XgbWrapper(xgb_params)
 with timer('Training Xgboost'):
xgb_oof_train, xgb_oof_test = get_oof(xgbWrapper, X_train_non_null.drop(['AdoptionSpeed'], axis=1), X_train_non_null['AdoptionSpeed'].values.astype(int), X_test_non_null, groups=rescuer_ids )<concatenate>
X_train_non_null.isnull().any().any() , X_test_non_null.isnull().any().any()
x_train_ensemble = np.concatenate(( et_oof_train, rf_oof_train, lgb_oof_train, xgb_oof_train), axis=1)
 x_test_ensemble = np.concatenate(( et_oof_test, rf_oof_test, lgb_oof_test, xgb_oof_test), axis=1 )<categorify>
def target_encoding(count_offset, X_te_train, X_te_valid, X_te_test, sigm_scale_divider=5):
categorical_column_list = ["Breed1", "Breed2", "Gender", "Color1", "Color2", "Color3", "Health", "State",
'sentiment_language']
target_encoding_columns = []
mean_all = X_te_train.AdoptionSpeed.mean()
for categ_column in categorical_column_list:
df_categorical = X_te_train[[categ_column, "AdoptionSpeed"]]
df_categorical = df_categorical.fillna(-1 ).astype(int)
 te_df = df_categorical.groupby(categ_column ).mean()
df_count = df_categorical.groupby(categ_column ).count()
sigmoid_scale =(count_offset - 1)/ sigm_scale_divider
smoothing_coef = 1 /(1 + np.exp(-(df_count - count_offset)/ sigmoid_scale))
target_enc_df = mean_all *(1 - smoothing_coef)+ te_df * smoothing_coef
target_enc_df = target_enc_df.add_prefix(f'TargetEncoding_{categ_column}_')
 X_te_train = pd.merge(X_te_train, target_enc_df.reset_index() , on=categ_column, how='left')
 X_te_valid = pd.merge(X_te_valid, target_enc_df.reset_index() , on=categ_column, how='left')
 X_te_test = pd.merge(X_te_test, target_enc_df.reset_index() , on=categ_column, how='left')
 return X_te_train, X_te_valid, X_te_test
ridge_params = {
'alpha':50.0,
'fit_intercept':True,
'normalize':False,
'copy_X':True,
'max_iter':None,
'tol':0.001,
'solver':'auto',
'random_state':1337
}
ridge = SklearnWrapper(clf=Ridge, seed=1337, params=ridge_params)
 final_oof_train = np.zeros(( ntrain,))
final_oof_test = np.zeros(( ntest,))
final_oof_test_skf = np.empty(( NFOLDS, ntest))
for i,(train_index, test_index)in enumerate(stratified_group_k_fold(x_train_ensemble, X_train_non_null['AdoptionSpeed'].values.astype(int), rescuer_ids, NFOLDS, 1337)) :
print('Training ridge regressor for fold: ', i + 1)
 x_tr = x_train_ensemble[train_index]
y_tr = X_train_non_null['AdoptionSpeed'].values[train_index]
x_te = x_train_ensemble[test_index]
y_te = X_train_non_null['AdoptionSpeed'].values[test_index]
print(x_tr.shape, y_tr.shape, x_te.shape, y_te.shape)
 ridge.train(x_tr, y_tr, x_val=x_te, y_val=y_te)
 final_oof_train[test_index] = ridge.predict(x_te)
 final_oof_test_skf[i, :] = ridge.predict(x_test_ensemble)
 final_oof_test[:] = final_oof_test_skf.mean(axis=0)
 final_oof_train = final_oof_train.reshape(-1, 1)
 final_oof_test = final_oof_test.reshape(-1, 1 )<compute_train_metric>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )
optR = OptimizedRounder()
optR.fit(final_oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(final_oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk )<set_options>
def rounder(y, thresholds=[0.5, 1.5, 2.5, 3.5]):
return np.digitize(y, thresholds)
 class OptimizedRounder(object):
def __init__(self):
self.coef_ = [0.5, 1.5, 2.5, 3.5]
def _loss(self, coef, X, y):
X_p = rounder(X, np.sort(coef))
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
coef = self.coef_
for it1 in range(10):
for idx in range(4):
golden1 = 2/(3+np.sqrt(5))
golden2 = 1 - golden1
a = 0.0 if idx-1 < 0 else coef[idx-1]
b = 4.0 if idx+1 > 3 else coef[idx+1]
coef[idx] = a +(b-a)*golden1
la = self._loss(coef, X, y)
 coef[idx] = a +(b-a)*golden2
lb = self._loss(coef, X, y)
 for it2 in range(20):
if la > lb:
a = a +(b-a)*golden1
coef[idx] = a +(b-a)*golden2
la = lb
lb = self._loss(coef, X, y)
 else:
b = a +(b-a)*golden2
coef[idx] = a +(b-a)*golden1
lb = la
la = self._loss(coef, X, y)
 self.coef_ = coef
def predict(self, X, coef):
return rounder(X, coef)
 def coefficients(self):
return self.coef_
gc.collect()<predict_on_test>
def stratified_group_k_fold(X, y, groups, k, seed=None):
labels_num = np.max(y)+ 1
y_counts_per_group = defaultdict(lambda: np.zeros(labels_num))
y_distr = Counter()
for label, g in zip(y, groups):
y_counts_per_group[g][label] += 1
y_distr[label] += 1
y_counts_per_fold = defaultdict(lambda: np.zeros(labels_num))
groups_per_fold = defaultdict(set)
 def eval_y_counts_per_fold(y_counts, fold):
y_counts_per_fold[fold] += y_counts
std_per_label = []
for label in range(labels_num):
label_std = np.std([y_counts_per_fold[i][label] / y_distr[label] for i in range(k)])
 std_per_label.append(label_std)
 y_counts_per_fold[fold] -= y_counts
return np.mean(std_per_label)
 groups_and_y_counts = list(y_counts_per_group.items())
 random.Random(seed ).shuffle(groups_and_y_counts)
 for g, y_counts in sorted(groups_and_y_counts, key=lambda x: -np.std(x[1])) :
best_fold = None
min_eval = None
for i in range(k):
fold_eval = eval_y_counts_per_fold(y_counts, i)
 if min_eval is None or fold_eval < min_eval:
min_eval = fold_eval
best_fold = i
y_counts_per_fold[best_fold] += y_counts
groups_per_fold[best_fold].add(g)
 all_groups = set(groups)
 for i in range(k):
train_groups = all_groups - groups_per_fold[i]
test_groups = groups_per_fold[i]
train_indices = [i for i, g in enumerate(groups)if g in train_groups]
test_indices = [i for i, g in enumerate(groups)if g in test_groups]
yield train_indices, test_indices
coefficients_ = coefficients.copy()
train_predictions = optR.predict(final_oof_train, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(final_oof_test, coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}' )<count_values>
train_gid = train.RescuerID.values
def run_xgb(params, X_train, X_test, cv):
n_splits = 10
verbose_eval = 10000
num_rounds = 5000
early_stop = 500
try:
kf = cv(n_splits=n_splits, shuffle=True)
 except:
kf = cv(n_splits=n_splits)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
importance_list = []
X_test_original = X_test.copy()
for train_idx, valid_idx in tqdm(kf.split(X_train, X_train['AdoptionSpeed'].values, train_gid)) :
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
X_tr, X_val, X_test = target_encoding(params['count_offset'], X_tr, X_val, X_test_original, params['sigm_scale_divider'])
 y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
importance_list.append(model.get_score(importance_type='gain'))
i += 1
return model, oof_train, oof_test.mean(axis=1)
 def run_xgb_stratified_group(params, X_train, X_test):
n_splits = 10
verbose_eval = 10000
num_rounds = 5000
early_stop = 500
oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
importance_list = []
X_test_original = X_test.copy()
for train_idx, valid_idx in tqdm(stratified_group_k_fold(X_train, X_train['AdoptionSpeed'].values.astype(np.int8), train_gid, k=n_splits)) :
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
X_tr, X_val, X_test = target_encoding(params['count_offset'], X_tr, X_val, X_test_original, params['sigm_scale_divider'])
 y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
importance_list.append(model.get_score(importance_type='gain'))
i += 1
return model, oof_train, oof_test.mean(axis=1 )
Counter(train_predictions )<count_values>
oof_train_list = []
oof_test_list = []
for i_seed in range(Config.num_seed_stratified):
print(i_seed)
 gc.collect()
xgb_params = {
'eval_metric': 'rmse',
'seed': i_seed,
'eta': 0.02042285921574787,
'gamma': 0.0006332233837072564,
'max_depth': 4,
'min_child_weight': 12,
'subsample': 0.8928595200024736,
'colsample_bytree': 0.3894222269910513,
'colsample_bylevel': 0.7,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
'count_offset': 40,
'sigm_scale_divider': 50
}
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null, StratifiedKFold)
 oof_train_list.append(oof_train)
 oof_test_list.append(oof_test)
 for i_seed in range(Config.num_seed_group):
print(i_seed)
 gc.collect()
xgb_params = {
'eval_metric': 'rmse',
'seed': i_seed,
'eta': 0.02042285921574787,
'gamma': 0.0006332233837072564,
'max_depth': 4,
'min_child_weight': 12,
'subsample': 0.8928595200024736,
'colsample_bytree': 0.3894222269910513,
'colsample_bylevel': 0.7,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
'count_offset': 40,
'sigm_scale_divider': 50
}
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null, GroupKFold)
 oof_train_list.append(oof_train)
 oof_test_list.append(oof_test)
 for i_seed in range(Config.num_seed_stratified_group):
print(i_seed)
 gc.collect()
xgb_params = {
'eval_metric': 'rmse',
'seed': i_seed,
'eta': 0.02042285921574787,
'gamma': 0.0006332233837072564,
'max_depth': 4,
'min_child_weight': 12,
'subsample': 0.8928595200024736,
'colsample_bytree': 0.3894222269910513,
'colsample_bylevel': 0.7,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
'count_offset': 40,
'sigm_scale_divider': 50
}
model, oof_train, oof_test = run_xgb_stratified_group(xgb_params, X_train_non_null, X_test_non_null)
 oof_train_list.append(oof_train)
 oof_test_list.append(oof_test)
 oof_train = np.mean(oof_train_list, axis=0)
 oof_test = np.mean(oof_test_list, axis=0)
Counter(test_predictions )<save_to_csv>
xgb_oof_train, xgb_oof_test = oof_train, oof_test
submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions})
 submission.to_csv('submission.csv', index=False)
 submission.head()<define_variables>
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk)
 coefficients_ = optR.coefficients()
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(oof_test, coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}' )
SUBMISSION = True
USE_RESCUER_CV = not SUBMISSION
LB = not SUBMISSION
SEED = 209321206
VERBOSE = 1<import_modules>
import lightgbm as lgb
START_TIME = pd.datetime.now() 
<set_options>
oof_train_list = []
oof_test_list = []
for i_seed in tqdm(range(3)) :
gc.collect()
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'rmse',
'eta': 0.01235408383482946,
'subsample': 0.90892648949676,
'colsample_bytree': 0.3595323228044939,
'min_split_gain': 0.0017213784304419208,
'min_child_weight': 2.489294481302874e-05,
'min_child_samples': 3,
'n_estimators': 1838,
'max_depth': 12,
'count_offset': 21.197455354403893,
'sigm_scale_divider': 49.03532266115622,
'bagging_freq': 4,
'verbose': -1,
'silent': 1,
'n_jobs': -1,
'seed': i_seed
}
model, oof_train, oof_test = run_lgbm(lgb_params, X_train_non_null, X_test_non_null, StratifiedKFold)
 oof_train_list.append(oof_train)
 oof_test_list.append(oof_test)
 for i_seed in tqdm(range(3)) :
gc.collect()
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'rmse',
'eta': 0.01235408383482946,
'subsample': 0.90892648949676,
'colsample_bytree': 0.3595323228044939,
'min_split_gain': 0.0017213784304419208,
'min_child_weight': 2.489294481302874e-05,
'min_child_samples': 3,
'n_estimators': 1838,
'max_depth': 12,
'count_offset': 21.197455354403893,
'sigm_scale_divider': 49.03532266115622,
'bagging_freq': 4,
'verbose': -1,
'silent': 1,
'n_jobs': -1,
'seed': i_seed
}
model, oof_train, oof_test = run_lgbm(lgb_params, X_train_non_null, X_test_non_null, GroupKFold)
 oof_train_list.append(oof_train)
 oof_test_list.append(oof_test)
 for i_seed in tqdm(range(3)) :
gc.collect()
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'rmse',
'eta': 0.01235408383482946,
'subsample': 0.90892648949676,
'colsample_bytree': 0.3595323228044939,
'min_split_gain': 0.0017213784304419208,
'min_child_weight': 2.489294481302874e-05,
'min_child_samples': 3,
'n_estimators': 1838,
'max_depth': 12,
'count_offset': 21.197455354403893,
'sigm_scale_divider': 49.03532266115622,
'bagging_freq': 4,
'verbose': -1,
'silent': 1,
'n_jobs': -1,
'seed': i_seed
}
model, oof_train, oof_test = run_lgbm_stratified_group(lgb_params, X_train_non_null, X_test_non_null)
 oof_train_list.append(oof_train)
 oof_test_list.append(oof_test)
 oof_train = np.mean(oof_train_list, axis=0)
 oof_test = np.mean(oof_test_list, axis=0 )
class KaggleHandler(StreamHandler):
def __init__(self):
StreamHandler.__init__(self)
 def emit(self, record):
msg_fmt = re.sub(r'[^0-9A-z\.\,\-\:\=]', ' ', str(self.format(record)))
 secs_passed =(pd.datetime.now() - START_TIME ).seconds
msg = '[%ss] %s' %(secs_passed, msg_fmt)
 os.system(f'echo {msg}')
 sys.stdout.write(msg + '
')
 logger = logging.getLogger(__name__)
 logger.handlers = []
logger.setLevel(logging.DEBUG)
 fmt = logging.Formatter('[%(asctime)s]: %(message)s')
 kh = KaggleHandler()
kh.setFormatter(fmt)
 logger.addHandler(kh )<define_variables>
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk)
 coefficients_ = optR.coefficients()
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(oof_test, coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}' )
logger.info('Seed set to %d' % SEED)
 np.random.seed(SEED )<data_type_conversions>
lgb_oof_train = oof_train
lgb_oof_test = oof_test
def qwk(predicted_ratings, actual_ratings, nb_ratings=5):

 assert len(actual_ratings)== len(predicted_ratings)
 actual_ratings = actual_ratings.astype(np.int64)
 predicted_ratings = predicted_ratings.astype(np.int64)
 conf_mtx = confusion_matrix(actual_ratings, predicted_ratings ).astype(np.float64)
 weight_mtx = np.zeros(( nb_ratings, nb_ratings))
for i in range(nb_ratings):
for j in range(nb_ratings):
weight_mtx[i][j] =(i-j)**2 /(( nb_ratings-1)**2)
 actual_ratings_hist = np.bincount(actual_ratings,minlength=nb_ratings)
 predicted_ratings_hist = np.bincount(predicted_ratings, minlength=nb_ratings)
 out_prod = np.outer(actual_ratings_hist, predicted_ratings_hist ).astype(np.float64)
 conf_mtx /= conf_mtx.sum()
out_prod /= out_prod.sum()
numerator =(conf_mtx * weight_mtx ).sum()
denominator =(out_prod * weight_mtx ).sum()
score =(1 -(numerator/denominator))
return score<find_best_model_class>
best_qwk = 0
for weight in np.arange(0, 1.01, 0.1):
print(weight)
 oof_train = weight * xgb_oof_train +(1 - weight)* lgb_oof_train
oof_test = weight * xgb_oof_test +(1 - weight)* lgb_oof_test
plot_pred(oof_train)
 plot_pred(oof_test)
 optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk)
 coefficients_ = optR.coefficients()
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(oof_test, coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}')
 if qwk > best_qwk:
best_weight = weight
best_qwk = qwk
best_test_predictions = test_predictions
print(best_weight, best_qwk )
<categorify><EOS> 
submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': best_test_predictions})
 submission.to_csv('submission.csv', index=False)
 submission.head()