Kaggle competition: PetFinder.my Adoption Prediction
<SOS> metric: QuadraticWeightedKappa Kaggle data source: petfinder.my-adoption-prediction<save_to_csv>
kappa_scorer = None
os.environ['USER'] = 'root'
os.system('pip install.. /input/xlearn/xlearn/xlearn-0.40a1/')
 print(os.listdir(".. /input"))
print(os.listdir(".. /input/petfinder-adoption-prediction/"))
print(os.listdir(".. /input/petfinder-adoption-prediction/train"))
print(os.listdir(".. /input/petfinder-adoption-prediction/test"))
sample_submission['target'] = test_preds
sample_submission.to_csv('submission.csv', index=False )<save_model>
train_df = pd.read_csv(".. /input/petfinder-adoption-prediction/train/train.csv")
 test_df = pd.read_csv(".. /input/petfinder-adoption-prediction/test/test.csv" )
cp.save('test_preds', test_preds)
 cp.save('train_oof', train_oof )<load_from_csv>
y = train_df["AdoptionSpeed"]
train = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')
 test = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv')
 test.insert(1, 'target', 0 )<categorify>
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv' )
features = [_f for _f in train if _f not in ['id', 'target']]
def factor_encoding(train, test):
assert sorted(train.columns)== sorted(test.columns)
 full = pd.concat([train, test], axis=0, sort=False)
 for f in full:
full[f], _ = pd.factorize(full[f])
 full[f] += 1
return full.iloc[:train.shape[0]], full.iloc[train.shape[0]:]
train_f, test_f = factor_encoding(train[features], test[features] )<categorify>
with open('.. /input/cat-and-dog-breeds-parameters/rating.json', 'r')as f:
ratings = json.load(f)
 cat_ratings = ratings['cat_breeds']
dog_ratings = ratings['dog_breeds']
breed_id = {}
class LibFFMEncoder(object):
def __init__(self):
self.encoder = 1
self.encoding = {}
def encode_for_libffm(self, row):
txt = f"{row[0]}"
for i, r in enumerate(row[1:]):
try:
txt += f' {i+1}:{self.encoding[(i, r)]}:1'
except KeyError:
self.encoding[(i, r)] = self.encoder
self.encoder += 1
txt += f' {i+1}:{self.encoding[(i, r)]}:1'
return txt
fold_ids = [
[trn_, val_] for(trn_, val_)in KFold(5,True,1 ).split(train)
 ]
for fold_,(trn_, val_)in enumerate(fold_ids):
encoder = LibFFMEncoder()
libffm_format_trn = pd.concat([train['target'].iloc[trn_], train_f.iloc[trn_]], axis=1 ).apply(
 lambda row: encoder.encode_for_libffm(row), raw=True, axis=1 
)
 libffm_format_val = pd.concat([train['target'].iloc[val_], train_f.iloc[val_]], axis=1 ).apply(
 lambda row: encoder.encode_for_libffm(row), raw=True, axis=1 
)
 print(train['target'].iloc[trn_].shape, train['target'].iloc[val_].shape, libffm_format_val.shape)
 libffm_format_trn.to_csv(f'libffm_trn_fold_{fold_+1}.txt', index=False, header=False)
 libffm_format_val.to_csv(f'libffm_val_fold_{fold_+1}.txt', index=False, header=False)
 encoder = LibFFMEncoder()
libffm_format_trn = pd.concat([train['target'], train_f], axis=1 ).apply(
 lambda row: encoder.encode_for_libffm(row), raw=True, axis=1 
)
 libffm_format_tst = pd.concat([test['target'], test_f], axis=1 ).apply(
 lambda row: encoder.encode_for_libffm(row), raw=True, axis=1 
)
 libffm_format_trn.to_csv(f'libffm_trn.txt', index=False, header=False)
 libffm_format_tst.to_csv(f'libffm_tst.txt', index=False, header=False )<load_from_csv>
pet_attributes = set(list(cat_ratings["Abyssinian"].keys())+ list(dog_ratings["German Shepherd Dog"].keys()))
!./ffm-train -p libffm_val_fold_1.txt -r 0.05 -l 0.00001 -k 50 -t 7 libffm_trn_fold_1.txt libffm_fold_1_model
!./ffm-predict libffm_val_fold_1.txt libffm_fold_1_model val_preds_fold_1.txt 
(
 log_loss(train['target'].iloc[fold_ids[0][1]], pd.read_csv('val_preds_fold_1.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[0][1]], pd.read_csv('val_preds_fold_1.txt', header=None ).values[:,0])
 )<load_from_csv>
for i in pet_attributes:
train_df[i] = 0
test_df[i] = 0
!./ffm-train -p libffm_val_fold_2.txt -r 0.05 -l 0.00001 -k 50 -t 7 libffm_trn_fold_2.txt libffm_fold_2_model
!./ffm-predict libffm_val_fold_2.txt libffm_fold_2_model val_preds_fold_2.txt 
(
 log_loss(train['target'].iloc[fold_ids[1][1]], pd.read_csv('val_preds_fold_2.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[1][1]], pd.read_csv('val_preds_fold_2.txt', header=None ).values[:,0])
 )<load_from_csv>
for id,name in zip(labels_breed.BreedID,labels_breed.BreedName):
breed_id[id] = str(name ).lower()
breed_names_1 = [i for i in cat_ratings.keys() ]
breed_names_2 = [i for i in dog_ratings.keys() ]
for i, id in enumerate(train_df['Breed1']):
if id in breed_id.keys() :
name = str(breed_id[id] ).lower()
if name in breed_names_1:
for key in cat_ratings[name].keys() :
train_df.loc[i, key] = cat_ratings[name][key]
if name in breed_names_2:
for key in dog_ratings[name].keys() :
train_df.loc[i, key] = dog_ratings[name][key]
for i, id in enumerate(test_df['Breed1']):
if id in breed_id.keys() :
str(breed_id[id] ).lower()
if name in breed_names_1:
for key in cat_ratings[name].keys() :
test_df.loc[i, key] = cat_ratings[name][key]
if name in breed_names_2:
for key in dog_ratings[name].keys() :
test_df.loc[i, key] = dog_ratings[name][key]
!./ffm-train -p libffm_val_fold_3.txt -r 0.05 -l 0.00001 -k 50 -t 7 libffm_trn_fold_3.txt libffm_fold_3_model
!./ffm-predict libffm_val_fold_3.txt libffm_fold_3_model val_preds_fold_3.txt 
(
 log_loss(train['target'].iloc[fold_ids[2][1]], pd.read_csv('val_preds_fold_3.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[2][1]], pd.read_csv('val_preds_fold_3.txt', header=None ).values[:,0])
 )<load_from_csv>
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print('num of train images files: {}'.format(len(train_image_files)))
 print('num of train metadata files: {}'.format(len(train_metadata_files)))
 print('num of train sentiment files: {}'.format(len(train_sentiment_files)))
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print('num of test images files: {}'.format(len(test_image_files)))
 print('num of test metadata files: {}'.format(len(test_metadata_files)))
 print('num of test sentiment files: {}'.format(len(test_sentiment_files)) )
!./ffm-train -p libffm_val_fold_4.txt -r 0.05 -l 0.00001 -k 50 -t 7 libffm_trn_fold_4.txt libffm_fold_4_model
!./ffm-predict libffm_val_fold_4.txt libffm_fold_4_model val_preds_fold_4.txt 
(
 log_loss(train['target'].iloc[fold_ids[3][1]], pd.read_csv('val_preds_fold_4.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[3][1]], pd.read_csv('val_preds_fold_4.txt', header=None ).values[:,0])
 )<load_from_csv>
train_df_ids = train_df[['PetID']]
print(train_df_ids.shape)
 train_df_imgs = pd.DataFrame(train_image_files)
 train_df_imgs.columns = ['image_filename']
train_imgs_pets = train_df_imgs['image_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 print(len(train_imgs_pets.unique()))
pets_with_images = len(np.intersect1d(train_imgs_pets.unique() , train_df_ids['PetID'].unique()))
print('fraction of pets with images: {:.3f}'.format(pets_with_images / train_df_ids.shape[0]))
train_df_ids = train_df[['PetID']]
train_df_metadata = pd.DataFrame(train_metadata_files)
 train_df_metadata.columns = ['metadata_filename']
train_metadata_pets = train_df_metadata['metadata_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 train_df_metadata = train_df_metadata.assign(PetID=train_metadata_pets)
 print(len(train_metadata_pets.unique()))
pets_with_metadatas = len(np.intersect1d(train_metadata_pets.unique() , train_df_ids['PetID'].unique()))
print('fraction of pets with metadata: {:.3f}'.format(pets_with_metadatas / train_df_ids.shape[0]))
train_df_ids = train_df[['PetID']]
train_df_sentiment = pd.DataFrame(train_sentiment_files)
 train_df_sentiment.columns = ['sentiment_filename']
train_sentiment_pets = train_df_sentiment['sentiment_filename'].apply(lambda x: x.split('/')[-1].split('.')[0])
 train_df_sentiment = train_df_sentiment.assign(PetID=train_sentiment_pets)
 print(len(train_sentiment_pets.unique()))
pets_with_sentiments = len(np.intersect1d(train_sentiment_pets.unique() , train_df_ids['PetID'].unique()))
print('fraction of pets with sentiment: {:.3f}'.format(pets_with_sentiments / train_df_ids.shape[0]))
!./ffm-train -p libffm_val_fold_5.txt -r 0.05 -l 0.00001 -k 50 -t 7 libffm_trn_fold_5.txt libffm_fold_5_model
!./ffm-predict libffm_val_fold_5.txt libffm_fold_5_model val_preds_fold_5.txt 
(
 log_loss(train['target'].iloc[fold_ids[4][1]], pd.read_csv('val_preds_fold_5.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[4][1]], pd.read_csv('val_preds_fold_5.txt', header=None ).values[:,0])
 )<load_from_csv>
test_df_ids = test_df[['PetID']]
print(test_df_ids.shape)
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 print(len(test_imgs_pets.unique()))
pets_with_images = len(np.intersect1d(test_imgs_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with images: {:.3f}'.format(pets_with_images / test_df_ids.shape[0]))
test_df_ids = test_df[['PetID']]
test_df_metadata = pd.DataFrame(test_metadata_files)
 test_df_metadata.columns = ['metadata_filename']
test_metadata_pets = test_df_metadata['metadata_filename'].apply(lambda x: x.split('/')[-1].split('-')[0])
 test_df_metadata = test_df_metadata.assign(PetID=test_metadata_pets)
 print(len(test_metadata_pets.unique()))
pets_with_metadatas = len(np.intersect1d(test_metadata_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with metadata: {:.3f}'.format(pets_with_metadatas / test_df_ids.shape[0]))
test_df_ids = test_df[['PetID']]
test_df_sentiment = pd.DataFrame(test_sentiment_files)
 test_df_sentiment.columns = ['sentiment_filename']
test_sentiment_pets = test_df_sentiment['sentiment_filename'].apply(lambda x: x.split('/')[-1].split('.')[0])
 test_df_sentiment = test_df_sentiment.assign(PetID=test_sentiment_pets)
 print(len(test_sentiment_pets.unique()))
pets_with_sentiments = len(np.intersect1d(test_sentiment_pets.unique() , test_df_ids['PetID'].unique()))
print('fraction of pets with sentiment: {:.3f}'.format(pets_with_sentiments / test_df_ids.shape[0]))
print('images and metadata distributions the same? {}'.format(
 np.all(test_metadata_pets == test_imgs_pets)) )
oof_preds = np.zeros(train.shape[0])
 for fold_,(_, val_)in enumerate(fold_ids):
oof_preds[val_] = pd.read_csv(f'val_preds_fold_{fold_+1}.txt', header=None ).values[:, 0]
oof_score = roc_auc_score(train['target'], oof_preds)
 print(oof_score )<save_to_csv>
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_metadata_file(self, filename):

 with open(filename, 'r', encoding='utf-8')as f:
metadata_file = json.load(f)
 return metadata_file
def open_sentiment_file(self, filename):

 with open(filename, 'r', encoding='utf-8')as f:
sentiment_file = json.load(f)
 return sentiment_file
def open_image_file(self, filename):

 image = np.asarray(Image.open(filename))
return image
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 if self.extract_sentiment_text:
file_sentences_text = [x['text']['content'] for x in file['sentences']]
file_sentences_text = self.sentence_sep.join(file_sentences_text)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns' ).sum()
file_sentences_sentiment = file_sentences_sentiment.add_prefix('document_' ).to_dict()
file_sentiment.update(file_sentences_sentiment)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
if self.extract_sentiment_text:
df_sentiment['text'] = file_sentences_text
df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations'][:int(len(file['labelAnnotations'])* 0.3)]
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = '.. /input/petfinder-adoption-prediction/{}_sentiment/{}.json'.format(mode, pet_id)
 try:
sentiment_file = pet_parser.open_sentiment_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob('.. /input/petfinder-adoption-prediction/{}_metadata/{}*.json'.format(mode, pet_id)))
 if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_metadata_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()
submission = test[['id']].copy()
submission['target'] = pd.read_csv('tst_preds.txt', header=None ).values[:,0]
submission.to_csv('libffm_prediction.csv', index=False )<load_from_csv>
aggregates = ['mean', 'sum', 'var', 'max', 'min']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(aggregates)
 train_sentiment_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(aggregates)
 test_sentiment_gr.columns = pd.Index(['{}_{}_{}'.format(
 prefix, c[0], c[1].upper())for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()
%matplotlib inline
test_features = pd.read_csv(".. /input/cat-in-the-dat-ii/test.csv")
 train_set = pd.read_csv(".. /input/cat-in-the-dat-ii/train.csv")
 train_targets = train_set.target
train_features = train_set.drop(['target'], axis=1)
 percentage = train_targets.mean() * 100
print("The percentage of ones in the training target is {:.2f}%".format(percentage))
train_features.head()<categorify>
train_proc = train_df.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test_df.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
columns = [col for col in train_features.columns if col != 'id']
df = train_features[columns]
train_encoded = pd.DataFrame()
skf = StratifiedKFold(n_splits=5,shuffle=True ).split(df, train_targets)
 for tr_in,fold_in in skf:
encoder = ce.WOEEncoder(cols=columns, handle_missing='return_nan')
 encoder.fit(df.iloc[tr_in,:], train_targets.iloc[tr_in])
 train_encoded = train_encoded.append(encoder.transform(df.iloc[fold_in,:]),ignore_index=False)
 train_encoded = train_encoded.sort_index()
encoder = ce.WOEEncoder(cols=columns, handle_missing='return_nan', handle_unknown='return_nan')
 encoder.fit(df, train_targets)
 test_encoded = encoder.transform(test_features[columns])
 train_encoded.head()<categorify>
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False, axis = 0)
 print('NaN structure:
{}'.format(np.sum(pd.isnull(X))))
new_train_encoded = train_encoded.copy()
new_test_encoded = test_encoded.copy()
for currently_encoding in columns:
tr_null = train_encoded.loc[train_encoded[currently_encoding].isnull() ].copy()
tr_not_null = train_encoded.loc[~train_encoded[currently_encoding].isnull() ].copy()
ts_null = test_encoded.loc[test_encoded[currently_encoding].isnull() ]
ts_not_null = test_encoded.loc[~test_encoded[currently_encoding].isnull() ]
temp_tr_feat = tr_not_null.drop(currently_encoding, axis=1 ).fillna(0)
 temp_tr_targ = tr_not_null[currently_encoding]
temp_val_feat = tr_null.drop(currently_encoding, axis=1 ).fillna(0)
 temp_test_feat = ts_null.drop(currently_encoding, axis=1 ).fillna(0)
 parameters = {
'n_estimators':30,
'max_depth':15,
'learning_rate':0.05,
'reg_lambda':0.03,
'reg_alpha':0.03,
'random_state':1728
}
regressor = XGBRegressor(**parameters)
 regressor.fit(temp_tr_feat, temp_tr_targ)
 temp_pred = pd.Series(regressor.predict(temp_val_feat))
temp_pred.index = temp_val_feat.index
new_train_encoded.loc[train_encoded[currently_encoding].isnull() , [currently_encoding]] = temp_pred
temp_pred = pd.Series(regressor.predict(temp_test_feat))
temp_pred.index = temp_test_feat.index
new_test_encoded.loc[test_encoded[currently_encoding].isnull() , [currently_encoding]] = temp_pred
new_train_encoded.info()<save_to_csv>
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
regressor = LogisticRegression(solver='lbfgs', max_iter=1000, C=0.6)
 regressor.fit(new_train_encoded, train_targets)
 probabilities = [pair[1] for pair in regressor.predict_proba(new_test_encoded)]
output = pd.DataFrame({'id': test_features['id'],
'target': probabilities})
 output.to_csv('submission.csv', index=False)
 output.describe()<load_from_csv>
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID')
 unique_count = X.groupby(['RescuerID'])['Breed1'].nunique().reset_index()
unique_count.columns = ['RescuerID', 'RescuerID_UNIQUE']
X_temp = X_temp.merge(unique_count, how='left', on='RescuerID')
 X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('<MISSING>' )
%matplotlib inline
test_features = pd.read_csv(".. /input/cat-in-the-dat-ii/test.csv")
 train_set = pd.read_csv(".. /input/cat-in-the-dat-ii/train.csv")
 train_targets = train_set.target
train_features = train_set.drop(['target'], axis=1)
 percentage = train_targets.mean() * 100
print("The percentage of ones in the training target is {:.2f}%".format(percentage))
train_features.head()<categorify>
n_components = 16
text_features = []
for i in X_text.columns:
print('generating features from: {}'.format(i))
svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 tfidf_col = TfidfVectorizer().fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('SVD_{}_'.format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp.reset_index(drop = True), text_features], axis=1)
 text_columns = ['metadata_annots_top_desc', 'sentiment_entities']
for i in text_columns:
X_temp = X_temp.drop(i, axis=1 )
columns = [col for col in train_features.columns if col != 'id']
woe_encoder = ce.WOEEncoder(cols=columns)
 woe_encoded_train = woe_encoder.fit_transform(train_features[columns], train_targets ).add_suffix('_woe')
 train_features = train_features.join(woe_encoded_train)
 woe_encoded_cols = woe_encoded_train.columns<groupby>
train_df = X_temp.loc[np.isfinite(X_temp.AdoptionSpeed), :]
test_df = X_temp.loc[~np.isfinite(X_temp.AdoptionSpeed), :]
test_df = test_df.drop(['AdoptionSpeed'], axis=1 )
df = train_features.copy()
df['target'] = train_targets
overall_number_of_ones = train_targets.sum()
overall_number_of_zeroes = 600000 - overall_number_of_ones
print("There are {} ones and {} zeroes in the training set".format(
 overall_number_of_ones, overall_number_of_zeroes 
))
grouped = pd.DataFrame()
grouped['Total'] = df.groupby('nom_0' ).id.count()
grouped['number of ones'] = df.groupby('nom_0' ).target.sum()
grouped['number of zeroes'] = grouped['Total'] - grouped['number of ones']
grouped['percentage of ones'] = grouped['number of ones'] / overall_number_of_ones
grouped['percentage of zeroes'] = grouped['number of zeroes'] / overall_number_of_zeroes
grouped['(% ones)>(% zeroes)'] = grouped['percentage of ones'] > grouped['percentage of zeroes']
grouped['weight of evidence'] = df.groupby('nom_0' ).nom_0_woe.mean()
grouped<groupby>
cat_cols = ['Breed1','Breed2', 'Gender', 'Color1', 'Color2', 'Color3', 'State', 'label_description']
grouped = pd.DataFrame()
grouped['Total'] = df.groupby('month' ).id.count()
grouped['number of ones'] = df.groupby('month' ).target.sum()
grouped['number of zeroes'] = grouped['Total'] - grouped['number of ones']
grouped['percentage of ones'] = grouped['number of ones'] / overall_number_of_ones
grouped['percentage of zeroes'] = grouped['number of zeroes'] / overall_number_of_zeroes
grouped['(% ones)>(% zeroes)'] = grouped['percentage of ones'] > grouped['percentage of zeroes']
grouped['weight of evidence'] = df.groupby('month' ).month_woe.mean()
grouped<categorify>
num_cols = ['Fee', 'Age', 'MaturitySize', 'FurLength', 'Vaccinated', 'Dewormed',
'Sterilized', 'Health', 'Quantity', 'VideoAmt', 'PhotoAmt',
'Type', 'doc_sent_mag', 'doc_sent_score', 'vertex_x', 'vertex_y',
'bounding_confidence', 'bounding_importance', 'dominant_blue', 'dominant_green',
'dominant_red', 'label_score', 'state_gdp', 'state_population',
'sentiment_sentiment_magnitude_MEAN', 'sentiment_sentiment_magnitude_SUM',
'sentiment_sentiment_score_MEAN', 'sentiment_sentiment_score_SUM',
'sentiment_sentiment_document_magnitude_MEAN', 'sentiment_sentiment_document_magnitude_SUM',
'sentiment_sentiment_document_score_MEAN', 'sentiment_sentiment_document_score_SUM',
'metadata_metadata_annots_score_MEAN', 'metadata_metadata_annots_score_SUM', 'metadata_metadata_color_score_MEAN',
'metadata_metadata_color_score_SUM', 'metadata_metadata_color_pixelfrac_MEAN', 'metadata_metadata_color_pixelfrac_SUM',
'metadata_metadata_crop_conf_MEAN', 'metadata_metadata_crop_conf_SUM', 'metadata_metadata_crop_importance_MEAN',
'metadata_metadata_crop_importance_SUM', 'RescuerID_COUNT',
'agg_mean_Age', 'agg_mean_Gender', 'agg_mean_Color1',
'agg_mean_Color2', 'agg_mean_Color3', 'agg_mean_FurLength', 'agg_mean_Vaccinated', 'agg_mean_Dewormed',
'agg_mean_Sterilized', 'agg_mean_Health', 'agg_mean_Quantity', 'agg_mean_Fee', 'agg_mean_PhotoAmt',
'agg_mean_VideoAmt', 'agg_std_Age', 'agg_std_Gender', 'agg_std_Color1', 'agg_std_Color2',
'agg_std_Color3', 'agg_std_FurLength', 'agg_std_Vaccinated', 'agg_std_Dewormed', 'agg_std_Sterilized',
'agg_std_Health', 'agg_std_Quantity', 'agg_std_Fee', 'agg_std_PhotoAmt', 'agg_std_VideoAmt', 'name_len',
'average_word_length', 'num_words', 'desc_char_length', 'RescuerID_UNIQUE']
def logreg_test(cols, encoder):
df = train_features[cols]
auc_scores = []
acc_scores = []
skf = StratifiedKFold(n_splits=6, shuffle=True ).split(df, train_targets)
 for train_id, valid_id in skf:
enc_tr = encoder.fit_transform(df.iloc[train_id,:], train_targets.iloc[train_id])
 enc_val = encoder.transform(df.iloc[valid_id,:])
 regressor = LogisticRegression(solver='lbfgs', max_iter=1000, C=0.6)
 regressor.fit(enc_tr, train_targets.iloc[train_id])
 acc_scores.append(regressor.score(enc_val, train_targets.iloc[valid_id]))
probabilities = [pair[1] for pair in regressor.predict_proba(enc_val)]
auc_scores.append(roc_auc_score(train_targets.iloc[valid_id], probabilities))
acc_scores = pd.Series(acc_scores)
 mean_acc = acc_scores.mean() * 100
print("Mean accuracy score: {:.3f}%".format(mean_acc))
auc_scores = pd.Series(auc_scores)
 mean_auc = auc_scores.mean() * 100
print("Mean AUC score: {:.3f}%".format(mean_auc))
print("Using Weight of Evidence Encoder")
 woe_encoder = ce.WOEEncoder(cols=columns)
 logreg_test(columns, woe_encoder)
 print("
Using Target Encoder")
 targ_encoder = ce.TargetEncoder(cols=columns, smoothing=0.2)
 logreg_test(columns, targ_encoder)
 print("
Using CatBoost Encoder")
 cb_encoder = ce.CatBoostEncoder(cols=columns)
 logreg_test(columns, cb_encoder )<categorify>
text_cols = ['Description']
encoder = ce.TargetEncoder(cols=columns, smoothing=0.2)
 encoded_train = encoder.fit_transform(train_features[columns], train_targets ).add_suffix('_targ_enc')
 train_features = train_features.join(encoded_train)
 encoder = ce.CatBoostEncoder(cols=columns)
 encoded_train = encoder.fit_transform(train_features[columns], train_targets ).add_suffix('_catboost')
 train_features = train_features.join(encoded_train)
 training_set = train_features.copy()
training_set['target'] = train_targets
corrmat = training_set.corr()
plt.subplots(figsize=(20,20))
sns.heatmap(corrmat, vmax=0.9, square=True )<categorify>
train_id = train_df['PetID']
test_id = test_df['PetID']
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in train_id:
try:
with open('.. /input/petfinder-adoption-prediction/train_sentiment/' + pet + '.json', 'r', encoding = "utf-8")as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 train_df.loc[:, 'doc_sent_mag'] = doc_sent_mag
train_df.loc[:, 'doc_sent_score'] = doc_sent_score
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in test_id:
try:
with open('.. /input/petfinder-adoption-prediction/test_sentiment/' + pet + '.json', 'r', encoding = "utf-8")as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 test_df.loc[:, 'doc_sent_mag'] = doc_sent_mag
test_df.loc[:, 'doc_sent_score'] = doc_sent_score
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in train_id:
try:
with open('.. /input/petfinder-adoption-prediction/train_metadata/' + pet + '-1.json', 'r', encoding = "utf-8")as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 print(nl_count)
 train_df.loc[:, 'vertex_x'] = vertex_xs
train_df.loc[:, 'vertex_y'] = vertex_ys
train_df.loc[:, 'bounding_confidence'] = bounding_confidences
train_df.loc[:, 'bounding_importance'] = bounding_importance_fracs
train_df.loc[:, 'dominant_blue'] = dominant_blues
train_df.loc[:, 'dominant_green'] = dominant_greens
train_df.loc[:, 'dominant_red'] = dominant_reds
train_df.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
train_df.loc[:, 'dominant_score'] = dominant_scores
train_df.loc[:, 'label_description'] = label_descriptions
train_df.loc[:, 'label_score'] = label_scores
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in test_id:
try:
with open('.. /input/petfinder-adoption-prediction/test_metadata/' + pet + '-1.json', 'r', encoding = "utf-8")as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 test_df.loc[:, 'vertex_x'] = vertex_xs
test_df.loc[:, 'vertex_y'] = vertex_ys
test_df.loc[:, 'bounding_confidence'] = bounding_confidences
test_df.loc[:, 'bounding_importance'] = bounding_importance_fracs
test_df.loc[:, 'dominant_blue'] = dominant_blues
test_df.loc[:, 'dominant_green'] = dominant_greens
test_df.loc[:, 'dominant_red'] = dominant_reds
test_df.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
test_df.loc[:, 'dominant_score'] = dominant_scores
test_df.loc[:, 'label_description'] = label_descriptions
test_df.loc[:, 'label_score'] = label_scores
df = train_features[columns]
train_encoded = pd.DataFrame()
skf = StratifiedKFold(n_splits=5,shuffle=True ).split(df, train_targets)
 for tr_in,fold_in in skf:
encoder = ce.WOEEncoder(cols=columns)
 encoder.fit(df.iloc[tr_in,:], train_targets.iloc[tr_in])
 train_encoded = train_encoded.append(encoder.transform(df.iloc[fold_in,:]),ignore_index=False)
 train_encoded = train_encoded.sort_index()
encoder = ce.WOEEncoder(cols=columns)
 encoder.fit(df, train_targets)
 test_encoded = encoder.transform(test_features[columns])
 regressor = LogisticRegression(solver='lbfgs', max_iter=1000, C=0.6)
 regressor.fit(train_encoded, train_targets)
 probabilities = [pair[1] for pair in regressor.predict_proba(test_encoded)]
output = pd.DataFrame({'id': test_features['id'],
'target': probabilities})
 output.to_csv('submission.csv', index=False)
 output.describe()<install_modules>
import tensorflow
!pip install --no-warn-conflicts -U -q scikit-learn==0.22.1 autoimpute impyute fancyimpute<import_modules>
from tqdm import tqdm
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
sns.set_style('whitegrid')
 warnings.filterwarnings('ignore')
 warnings.simplefilter('ignore')
 pd.set_option('display.max_columns', 1000)
 pd.set_option('display.max_rows', 500)
 SEED = 2020
SPLITS = 25<define_variables>
num_words = 200000
maxlen = 200
embed_size = 300
def set_seed(seed=SEED):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 set_seed()<load_from_csv>
test_df = test_df.reset_index(drop = True )
base = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')
 baseTe = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv')
 baseTe['target'] = -1
score = dict()<create_dataframe>
train_df.loc[:, "Description"] = pd.read_csv(".. /input/petfinder-adoption-prediction/train/train.csv")["Description"]
test_df.loc[:, "Description"] = pd.read_csv(".. /input/petfinder-adoption-prediction/test/test.csv")["Description"]
pd.DataFrame(base.isna().sum(axis=0)/len(base), columns=['missing percent'] ).sort_values('missing percent', ascending=False ).T<create_dataframe>
train_df['Description'] = train_df['Description'].astype(str ).fillna('no text')
 test_df['Description'] = test_df['Description'].astype(str ).fillna('no text')
 train_df['Description1'] = train_df['Description'].astype(str ).fillna('no text')
 test_df['Description1'] = test_df['Description'].astype(str ).fillna('no text' )
def null_analysis(df):

 null_cnt = df.isnull().sum()
null_cnt = null_cnt[null_cnt!=0]
null_percent = null_cnt / len(df)* 100
null_table = pd.concat([pd.DataFrame(null_cnt), pd.DataFrame(null_percent)], axis=1)
 null_table.columns = ['counts', 'percentage']
null_table.sort_values('counts', ascending=False, inplace=True)
 return null_table<define_variables>
train_df['Description1'] = train_df['Description1'].astype(str ).fillna('no text')
 test_df['Description1'] = test_df['Description1'].astype(str ).fillna('no text' )
score = dict()<categorify>
print("Fitting tokenizer...")
 tokenizer = Tokenizer(num_words=num_words)
 tokenizer.fit_on_texts(train_df['Description'].values.tolist() + test_df['Description'].values.tolist() )
def SiavashMapping(df):
bin_3_mapping = {'T':1 , 'F':0}
bin_4_mapping = {'Y':1 , 'N':0}
nom_0_mapping = {'Red' : 0, 'Blue' : 1, 'Green' : 2}
nom_1_mapping = {'Trapezoid' : 0, 'Star' : 1, 'Circle': 2, 'Triangle' : 3, 'Polygon' : 4, 'Square': 5}
nom_2_mapping = {'Hamster' : 0 , 'Axolotl' : 1, 'Lion' : 2, 'Dog' : 3, 'Cat' : 4, 'Snake' : 5}
nom_3_mapping = {'Russia' : 0, 'Canada' : 1, 'Finland' : 2, 'Costa Rica' : 3, 'China' : 4, 'India' : 5}
nom_4_mapping = {'Bassoon' : 0, 'Theremin' : 1, 'Oboe' : 2, 'Piano' : 3}
nom_5_mapping = dict(zip(( df.nom_5.dropna().unique()), range(len(( df.nom_5.dropna().unique())))))
 nom_6_mapping = dict(zip(( df.nom_6.dropna().unique()), range(len(( df.nom_6.dropna().unique())))))
 nom_7_mapping = dict(zip(( df.nom_7.dropna().unique()), range(len(( df.nom_7.dropna().unique())))))
 nom_8_mapping = dict(zip(( df.nom_8.dropna().unique()), range(len(( df.nom_8.dropna().unique())))))
 nom_9_mapping = dict(zip(( df.nom_9.dropna().unique()), range(len(( df.nom_9.dropna().unique())))))
 ord_1_mapping = {'Novice' : 0, 'Contributor' : 1, 'Expert' : 2, 'Master': 3, 'Grandmaster': 4}
ord_2_mapping = { 'Freezing': 0, 'Cold': 1, 'Warm' : 2, 'Hot': 3, 'Boiling Hot' : 4, 'Lava Hot' : 5}
ord_3_mapping = {'a':0, 'b':1, 'c':2 ,'d':3 ,'e':4, 'f':5, 'g':6, 'h':7, 'i':8, 'j':9, 'k':10, 'l':11, 'm':12, 'n':13, 'o':14}
ord_4_mapping = {'A':0, 'B':1, 'C':2, 'D':3, 'E':4, 'F':5, 'G':6, 'H':7, 'I':8, 'J':9, 'K':10,'L':11,'M':12,
'N':13,'O':14,'P':15,'Q':16,'R':17,'S':18,'T':19,'U':20,'V':21,'W':22,'X':23,'Y':24,'Z':25}
sorted_ord_5 = sorted(df.ord_5.dropna().unique())
 ord_5_mapping = dict(zip(sorted_ord_5, range(len(sorted_ord_5))))
df['bin_3'] = df.loc[df.bin_3.notnull() , 'bin_3'].map(bin_3_mapping)
 df['bin_4'] = df.loc[df.bin_4.notnull() , 'bin_4'].map(bin_4_mapping)
 df['nom_0'] = df.loc[df.nom_0.notnull() , 'nom_0'].map(nom_0_mapping)
 df['nom_1'] = df.loc[df.nom_1.notnull() , 'nom_1'].map(nom_1_mapping)
 df['nom_2'] = df.loc[df.nom_2.notnull() , 'nom_2'].map(nom_2_mapping)
 df['nom_3'] = df.loc[df.nom_3.notnull() , 'nom_3'].map(nom_3_mapping)
 df['nom_4'] = df.loc[df.nom_4.notnull() , 'nom_4'].map(nom_4_mapping)
 df['nom_5'] = df.loc[df.nom_5.notnull() , 'nom_5'].map(nom_5_mapping)
 df['nom_6'] = df.loc[df.nom_6.notnull() , 'nom_6'].map(nom_6_mapping)
 df['nom_7'] = df.loc[df.nom_7.notnull() , 'nom_7'].map(nom_7_mapping)
 df['nom_8'] = df.loc[df.nom_8.notnull() , 'nom_8'].map(nom_8_mapping)
 df['nom_9'] = df.loc[df.nom_9.notnull() , 'nom_9'].map(nom_9_mapping)
 df['ord_1'] = df.loc[df.ord_1.notnull() , 'ord_1'].map(ord_1_mapping)
 df['ord_2'] = df.loc[df.ord_2.notnull() , 'ord_2'].map(ord_2_mapping)
 df['ord_3'] = df.loc[df.ord_3.notnull() , 'ord_3'].map(ord_3_mapping)
 df['ord_4'] = df.loc[df.ord_4.notnull() , 'ord_4'].map(ord_4_mapping)
 df['ord_5'] = df.loc[df.ord_5.notnull() , 'ord_5'].map(ord_5_mapping)
 return df<categorify>
train_df['Description'] = tokenizer.texts_to_sequences(train_df['Description'])
 test_df['Description'] = tokenizer.texts_to_sequences(test_df['Description'] )
def AntMapping(df, ordinal):
ord_maps = {
'ord_0': {val: i for i, val in enumerate([1, 2, 3])},
'ord_1': {
val: i
for i, val in enumerate(
 ['Novice', 'Contributor', 'Expert', 'Master', 'Grandmaster'] 
)
 },
'ord_2': {
val: i
for i, val in enumerate(
 ['Freezing', 'Cold', 'Warm', 'Hot', 'Boiling Hot', 'Lava Hot'] 
)
 },
**{col: {val: i for i, val in enumerate(sorted(df[col].dropna().unique())) } for col in ['ord_3', 'ord_4', 'ord_5', 'day', 'month']},
}
ord_cols = pd.concat([df[col].map(ord_map ).fillna(max(ord_map.values())//2 ).astype('float32')for col, ord_map in ord_maps.items() ], axis=1)
 ord_cols /= ord_cols.max()
ord_sqr = 4*(ord_cols - 0.5)**2
ord_cols_sqr = [feat+'_sqr' for feat in ordinal]
df[ordinal] = ord_cols
df[ord_cols_sqr] = ord_sqr
return df<categorify>
avg_word_length = []
desc_length = []
number_words = []
for desc in train_df["Description1"]:
desc_length.append(len(desc))
words = desc.split()
number_words.append(len(words))
word_len = 0
for word in words:
word_len += len(word)
 avg_word_length.append(word_len / len(words))
train_df["average_word_length"] = avg_word_length
train_df["num_words"] = number_words
train_df["desc_char_length"] = desc_length
def CountEncoding(df, cols, df_test=None):
for col in cols:
frequencies = df[col].value_counts().reset_index()
df_values = df[[col]].merge(frequencies, how='left', left_on=col, right_on='index' ).iloc[:,-1].values
df[col+'_counts'] = df_values
if df_test is not None:
df_test_values = df_test[[col]].merge(frequencies, how='left', left_on=col, right_on='index' ).fillna(1 ).iloc[:,-1].values
df_test[col+'_counts'] = df_test_values
count_cols = [col+'_counts' for col in cols]
if df_test is not None:
return df, df_test, count_cols
else:
return df, count_cols 
<categorify>
avg_word_length = []
desc_length = []
number_words = []
for desc in test_df["Description1"]:
desc_length.append(len(desc))
words = desc.split()
number_words.append(len(words))
word_len = 0
for word in words:
word_len += len(word)
 avg_word_length.append(word_len / len(words))
test_df["average_word_length"] = avg_word_length
test_df["num_words"] = number_words
test_df["desc_char_length"] = desc_length
def YurgensenMapping(df):
def TLC(s):
s = str(s)
 return(((ord(s[0])) -64)*52+(( ord(s[1])) -64)-6)
 df['ord_0_ord_2'] = df['ord_0'].astype('str')+df['ord_2'].astype('str')
 df['ord_0'] = df['ord_0'].fillna(2.01)
 df.loc[df.ord_2=='Freezing', 'ord_2'] = 0
df.loc[df.ord_2=='Cold', 'ord_2'] = 1
df.loc[df.ord_2=='Warm', 'ord_2'] = 2
df.loc[df.ord_2=='Hot', 'ord_2'] = 3
df.loc[df.ord_2=='Boiling Hot', 'ord_2'] = 4
df.loc[df.ord_2=='Lava Hot', 'ord_2'] = 5
df['ord_2'] = df['ord_2'].fillna(2.37)
 df.loc[df.ord_1=='Novice', 'ord_1'] = 0
df.loc[df.ord_1=='Contributor', 'ord_1'] = 1
df.loc[df.ord_1=='Expert', 'ord_1'] = 2
df.loc[df.ord_1=='Master', 'ord_1'] = 3
df.loc[df.ord_1=='Grandmaster', 'ord_1'] = 4
df['ord_1'] = df['ord_1'].fillna(1.86)
 df['ord_5'] = df.loc[df.ord_5.notnull() , 'ord_5'].apply(lambda x: TLC(x))
df['ord_5'] = df['ord_5'].fillna('Zx' ).apply(lambda x: TLC(x))
df['ord_5'] = df['ord_5'].rank()
df['ord_3'] = df.loc[df.ord_3.notnull() , 'ord_3'].apply(lambda x: ord(str(x)) -96)
 df['ord_3'] = df['ord_3'].fillna(8.44)
 df['ord_4'] = df.loc[df.ord_4.notnull() , 'ord_4'].apply(lambda x: ord(str(x)) -64)
 df['ord_4'] = df['ord_4'].fillna(14.31)
 return df 
<categorify>
name_len = []
for name in train_df["Name"]:
try:
name_len.append(len(name))
except:
name_len.append(-1)
 train_df["name_len"] = name_len
def RidgeClf(train, test, ordinal, ohe, scaler, seed, splits, drop_idx=None, dimreducer=None):
y_train = train['target'].values.copy()
train_length = train.shape[0]
test['target'] = -1
data = pd.concat([train, test], axis=0 ).reset_index(drop=True)
 X_ohe = pd.get_dummies(data[ohe],columns=ohe,drop_first=True,dummy_na=True,sparse=True, dtype='int8' ).sparse.to_coo()
if dimreducer is not None:
X_ohe = sp.sparse.csr_matrix(dimreducer.fit_transform(X_ohe))
gc.collect()
if ordinal is not None:
if scaler is not None:
X_ord = scaler.fit_transform(data[ordinal])
 else:
X_ord = data[ordianl].values
data_ = sp.sparse.hstack([X_ohe, X_ord] ).tocsr()
else:
data_ = sp.sparse.hstack([X_ohe] ).tocsr()
train = data_[:train_length]
test = data_[train_length:]
model = RidgeClassifier(alpha=152.5)
 skf = StratifiedKFold(n_splits=splits, shuffle=True, random_state=seed)
 oof = np.zeros(( train.shape[0],))
y_pred = np.zeros(( test.shape[0],))
for tr_ind, val_ind in skf.split(train, y_train):
if drop_idx is not None:
idx = list(set(tr_ind)-set(drop_idx))
X_tr, X_val = train[idx], train[val_ind]
y_tr, y_val = y_train[idx], y_train[val_ind]
else:
X_tr, X_val = train[tr_ind], train[val_ind]
y_tr, y_val = y_train[tr_ind], y_train[val_ind]
train_set = {'X':X_tr, 'y':y_tr}
val_set = {'X':X_val, 'y':y_val}
model.fit(train_set['X'], train_set['y'])
 oof[val_ind] = model.decision_function(val_set['X'])
 y_pred += model.decision_function(test)/ splits
oof_auc_score = roc_auc_score(y_train, oof)
 oof = rankdata(oof)/len(oof)
 y_pred = rankdata(y_pred)/len(y_pred)
 return oof, y_pred, oof_auc_score 
<prepare_x_and_y>
name_len = []
for name in test_df["Name"]:
try:
name_len.append(len(name))
except:
name_len.append(-1)
 test_df["name_len"] = name_len
def LogRegClf(train, test, ordinal, ohe, scaler, seed, splits, drop_idx=None, dimreducer=None):
params = {
'fit_intercept' : True,
'random_state': SEED,
'penalty' : 'l2',
'verbose' : 0,
'solver' : 'lbfgs',
'max_iter' : 1000,
'n_jobs' : 4,
'C' : 0.05,
}
y_train = train['target'].values.copy()
train_length = train.shape[0]
test['target'] = -1
data = pd.concat([train, test], axis=0 ).reset_index(drop=True)
 X_ohe = pd.get_dummies(data[ohe],columns=ohe,drop_first=True,dummy_na=True,sparse=True, dtype='int8' ).sparse.to_coo()
if dimreducer is not None:
X_ohe = sp.sparse.csr_matrix(dimreducer.fit_transform(X_ohe))
gc.collect()
if ordinal is not None:
if scaler is not None:
X_ord = scaler.fit_transform(data[ordinal])
 else:
X_ord = data[ordinal].values
data_ = sp.sparse.hstack([X_ohe, X_ord] ).tocsr()
else:
data_ = sp.sparse.hstack([X_ohe] ).tocsr()
train = data_[:train_length]
test = data_[train_length:]
model = LogisticRegression(**params)
 skf = StratifiedKFold(n_splits=splits, shuffle=True, random_state=seed)
 oof = np.zeros(( train.shape[0],))
y_pred = np.zeros(( test.shape[0],))
for tr_ind, val_ind in skf.split(train, y_train):
if drop_idx is not None:
idx = list(set(tr_ind)-set(drop_idx))
X_tr, X_val = train[idx], train[val_ind]
y_tr, y_val = y_train[idx], y_train[val_ind]
else:
X_tr, X_val = train[tr_ind], train[val_ind]
y_tr, y_val = y_train[tr_ind], y_train[val_ind]
train_set = {'X':X_tr, 'y':y_tr}
val_set = {'X':X_val, 'y':y_val}
model.fit(train_set['X'], train_set['y'])
 oof[val_ind] = model.predict_proba(val_set['X'])[:, 1]
y_pred += model.predict_proba(test)[:, 1] / splits
oof_auc_score = roc_auc_score(y_train, oof)
 oof = rankdata(oof)/len(oof)
 y_pred = rankdata(y_pred)/len(y_pred)
 return oof, y_pred, oof_auc_score 
<categorify>
state_gdp = {
41336: 116.679,
41325: 40.596,
41367: 23.02,
41401: 190.075,
41415: 5.984,
41324: 37.274,
41332: 42.389,
41335: 52.452,
41330: 67.629,
41380: 5.642,
41327: 81.284,
41345: 80.167,
41342: 121.414,
41326: 280.698,
41361: 32.270
}
state_population = {
41336: 33.48283,
41325: 19.47651,
41367: 15.39601,
41401: 16.74621,
41415: 0.86908,
41324: 8.21110,
41332: 10.21064,
41335: 15.00817,
41330: 23.52743,
41380: 2.31541,
41327: 15.61383,
41345: 32.06742,
41342: 24.71140,
41326: 54.62141,
41361: 10.35977
}
train_df["state_gdp"] = train_df.State.map(state_gdp)
 train_df["state_population"] = train_df.State.map(state_population)
 test_df["state_gdp"] = test_df.State.map(state_gdp)
 test_df["state_population"] = test_df.State.map(state_population )
train = base.copy()
test = baseTe.copy()
features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]
ordinal = [feat for feat in features if feat not in ohe]
train[features] = SiavashMapping(train[features])
 test[features] = SiavashMapping(test[features])
 imp = SimpleImputer(strategy='constant')
 train[features] = imp.fit_transform(train[features])
 test[features] = imp.transform(test[features])
 train[features] = train[features].astype(np.int16)
 test[features] = test[features].astype(np.int16)
 scaler = RobustScaler(quantile_range=(10.0, 90.0))
oof1, pred1, score['Constant'] = RidgeClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)
 print(f'AUC score for Constant Imputation : {score["Constant"]}' )<categorify>
word_index = tokenizer.word_index
nb_words = min(num_words, len(word_index))
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= nb_words: continue
try:
embedding_vector = embeddings_index[word]
except:
embedding_vector = None
if embedding_vector is not None: embedding_matrix[i] = embedding_vector
train = base.copy()
test = baseTe.copy()
features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]
ordinal = [feat for feat in features if feat not in ohe]
train[features] = SiavashMapping(train[features])
 test[features] = SiavashMapping(test[features])
 imp = SimpleImputer(strategy='constant')
 train[ordinal] = imp.fit_transform(train[ordinal])
 test[ordinal] = imp.transform(test[ordinal])
 train[ordinal] = train[ordinal].astype(np.int16)
 test[ordinal] = test[ordinal].astype(np.int16)
 scaler = RobustScaler(quantile_range=(10.0, 90.0))
oof2, pred2, score['Constant-OrdinalOnly'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)
 print(f'AUC score for Constant Imputation of ordinal columns: {score["Constant-OrdinalOnly"]}' )<categorify>
for col in cat_cols:
le = LabelEncoder()
le.fit(train_df[col].tolist() + test_df[col].tolist())
 train_df[col] = le.transform(train_df[col].tolist())
 test_df[col] = le.transform(test_df[col].tolist())
 embed_sizes = [len(set(list(train_df[col].unique())+ list(test_df[col].unique())))+ 1 for col in cat_cols]
train = base.copy()
test = baseTe.copy()
features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = [feat for feat in features if feat not in ['ord_3', 'ord_4', 'ord_5']]
ordinal = [feat for feat in features if feat not in ohe]
train[features] = SiavashMapping(train[features])
 test[features] = SiavashMapping(test[features])
 imp = SimpleImputer(strategy='constant')
 train[ordinal] = imp.fit_transform(train[ordinal])
 test[ordinal] = imp.transform(test[ordinal])
 train[ordinal] = train[ordinal].astype(np.int16)
 test[ordinal] = test[ordinal].astype(np.int16)
 scaler = RobustScaler(quantile_range=(10.0, 90.0))
oof3, pred3, score['Constant-Ord4Only'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)
 print(f'AUC score for Constant Imputation of ord_4 column: {score["Constant-Ord4Only"]}' )<normalization>
agg_list = ["Age", "Gender", "FurLength", "Vaccinated", "Dewormed", "Sterilized",
"Health", "Quantity", "Fee", "PhotoAmt", "VideoAmt", "Color1", "Color2", "Color3", "vertex_y", "label_score"]
train_df[agg_list]
train = base.copy()
test = baseTe.copy()
features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = features
ordinal = None
train[features] = SiavashMapping(train[features])
 test[features] = SiavashMapping(test[features])
 scaler = RobustScaler(quantile_range=(10.0, 90.0))
oof4, pred4, score['CompleteOHE'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)
 print(f'AUC score for No imputation just OHE: {score["CompleteOHE"]}' )<categorify>
for col in agg_list:
means = train_df.groupby("RescuerID")[col].mean()
train_df["agg_mean_" + col] = train_df["RescuerID"].map(means)
 means = test_df.groupby("RescuerID")[col].mean()
test_df["agg_mean_" + col] = test_df["RescuerID"].map(means)
 num_cols.append("agg_mean_" + col)
 for col in agg_list:
stds = train_df.groupby("RescuerID")[col].std()
stds = stds.fillna(stds.mean())
 train_df["agg_std_" + col] = train_df["RescuerID"].map(stds)
 stds = test_df.groupby("RescuerID")[col].std()
test_df["agg_std_" + col] = test_df["RescuerID"].map(stds)
 num_cols.append("agg_std_" + col )
train = base.copy()
test = baseTe.copy()
features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = features
ordinal = None
train[features] = SiavashMapping(train[features])
 test[features] = SiavashMapping(test[features])
 imp = SimpleImputer(strategy='constant')
 train[features] = imp.fit_transform(train[features])
 test[features] = imp.transform(test[features])
 train[features] = train[features].astype(np.int16)
 test[features] = test[features].astype(np.int16)
 scaler = RobustScaler(quantile_range=(10.0, 90.0))
oof5, pred5, score['Constant-CompleteOHE'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)
 print(f'AUC score for Constant Imputation with complete OHE: {score["Constant-CompleteOHE"]}' )<categorify>
train_df_copy = deepcopy(train_df)
 test_df_copy = deepcopy(test_df )
train = base.copy()
test = baseTe.copy()
features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = features
ordinal = None
train[features] = SiavashMapping(train[features])
 test[features] = SiavashMapping(test[features])
 imp = SingleImputer(strategy='mode')
 train[features] = imp.fit_transform(train[features])
 test[features] = imp.transform(test[features])
 train[features] = train[features].astype(np.int16)
 test[features] = test[features].astype(np.int16)
 scaler = RobustScaler(quantile_range=(10.0, 90.0))
oof6, pred6, score['Mode-CompleteOHE'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)
 print(f'AUC score for Mode Imputation with complete OHE: {score["Mode-CompleteOHE"]}' )<categorify>
img_size = 256
batch_size = 16
pet_ids = train_df['PetID'].values
n_batches = len(pet_ids)// batch_size +(len(pet_ids)% batch_size != 0)
 def resize_to_square(im):
old_size = im.shape[:2]
ratio = float(img_size)/max(old_size)
 new_size = tuple([int(x*ratio)for x in old_size])
 im = cv2.resize(im,(new_size[1], new_size[0]))
delta_w = img_size - new_size[1]
delta_h = img_size - new_size[0]
top, bottom = delta_h//2, delta_h-(delta_h//2)
 left, right = delta_w//2, delta_w-(delta_w//2)
 color = [0, 0, 0]
new_im = cv2.copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT,value=color)
 return new_im
def load_image(path, pet_id):
image = cv2.imread(f'{path}{pet_id}-1.jpg')
 new_image = resize_to_square(image)
 new_image = preprocess_input(new_image)
 return new_image
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp, include_top = False, weights = None)
 backbone.load_weights(".. /input/densenet-keras/DenseNet-BC-121-32-no-top.h5")
 print("weights loaded succesfully")
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out)
 features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
train_feats = pd.DataFrame.from_dict(features, orient='index' ).values
n_components = 32
svd = TruncatedSVD(n_components=n_components)
 svd.fit(train_feats)
 train_feats = svd.transform(train_feats)
 train_feats = pd.DataFrame(train_feats, columns=['img_svd1_{}'.format(i)for i in range(n_components)])
 train_feats.to_csv('train_img_features.csv')
 train_df = pd.concat([train_df, train_feats], axis = 1)
 pet_ids = test_df['PetID'].values
n_batches = len(pet_ids)// batch_size +(len(pet_ids)% batch_size != 0)
 features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/test_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
test_feats = pd.DataFrame.from_dict(features, orient='index' ).values
test_feats = svd.transform(test_feats)
 test_feats = pd.DataFrame(test_feats, columns=['img_svd1_{}'.format(i)for i in range(n_components)])
 test_feats.to_csv('test_img_features.csv')
 test_df = pd.concat(( test_df.reset_index(drop = True), test_feats), axis=1)
 print(train_df.shape, test_df.shape )
train = base.copy()
test = baseTe.copy()
features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]
ordinal = [feat for feat in features if feat not in ohe]
train[features] = SiavashMapping(train[features])
 test[features] = SiavashMapping(test[features])
 imp = SingleImputer(strategy='mean')
 train[ordinal] = imp.fit_transform(train[ordinal])
 test[ordinal] = imp.transform(test[ordinal])
 train[ordinal] = train[ordinal].astype(np.int16)
 test[ordinal] = test[ordinal].astype(np.int16)
 scaler = RobustScaler(quantile_range=(10.0, 90.0))
oof7, pred7, score['Mean-OHE'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)
 print(f'AUC score for Mean Imputation with OHE on non-ordinal columns: {score["Mean-OHE"]}' )<normalization>
catvsdog = load_model(".. /input/kerascatvsdog/best.hd5")
 img_size = 299
batch_size = 16
pet_ids = train_df['PetID'].values
n_batches = len(pet_ids)// batch_size +(len(pet_ids)% batch_size != 0)
 def resize_to_square(im):
old_size = im.shape[:2]
ratio = float(img_size)/max(old_size)
 new_size = tuple([int(x*ratio)for x in old_size])
 im = cv2.resize(im,(new_size[1], new_size[0]))
delta_w = img_size - new_size[1]
delta_h = img_size - new_size[0]
top, bottom = delta_h//2, delta_h-(delta_h//2)
 left, right = delta_w//2, delta_w-(delta_w//2)
 color = [0, 0, 0]
new_im = cv2.copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT,value=color)
 return new_im
def load_image(path, pet_id):
image = cv2.imread(f'{path}{pet_id}-1.jpg')
 new_image = resize_to_square(image)
 new_image = preprocess_input(new_image)
 return new_image
train = base.copy()
test = baseTe.copy()
train = YurgensenMapping(train)
 test = YurgensenMapping(test)
 features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]
ordinal = [feat for feat in features if feat not in ohe]
scaler = RobustScaler(quantile_range=(10.0, 90.0))
oof8, pred8, score['Yurgensen'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)
 print(f'AUC score for Yurgensen ordinal-Mapping : {score["Yurgensen"]}' )<feature_engineering>
backbone = catvsdog
out = backbone.layers[-2].output
m = Model(backbone.input, out)
 features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
train_feats = pd.DataFrame.from_dict(features, orient='index' ).values
n_components = 32
svd = TruncatedSVD(n_components=n_components)
 svd.fit(train_feats)
 train_feats = svd.transform(train_feats)
 train_feats = pd.DataFrame(train_feats, columns=['img_svd2_{}'.format(i)for i in range(n_components)])
 train_feats.to_csv('train_img_features.csv')
 train_df = pd.concat([train_df, train_feats], axis = 1)
 pet_ids = test_df['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/test_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
test_feats = pd.DataFrame.from_dict(features, orient='index' ).values
test_feats = svd.transform(test_feats)
 test_feats = pd.DataFrame(test_feats, columns=['img_svd2_{}'.format(i)for i in range(n_components)])
 test_feats.to_csv('test_img_features.csv')
 test_df = pd.concat(( test_df.reset_index(drop = True), test_feats), axis=1)
 print(train_df.shape, test_df.shape )
train = base.copy()
test = baseTe.copy()
features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]
ordinal = [feat for feat in features if feat not in ohe] + ['day', 'month']
train = AntMapping(train, ordinal)
 test = AntMapping(test , ordinal)
 scaler = None
oof9, pred9, score['Ant'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)
 print(f'AUC score for Max Imputation of ordinals with OHE on non-ordinal columns: {score["Ant"]}' )<categorify>
n_components = 32
train_desc = train_df.Description1.fillna("none" ).values
test_desc = test_df.Description1.fillna("none" ).values
print(test_desc.shape)
 tfv = TfidfVectorizer(min_df=50, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1, 
)
 tfv.fit(list(train_desc))
X = tfv.transform(train_desc)
 X_test = tfv.transform(test_desc)
 svd = TruncatedSVD(n_components=n_components)
 svd.fit(X)
 X = svd.transform(X)
 X = pd.DataFrame(X, columns=['svd1_{}'.format(i)for i in range(n_components)])
 train_df = pd.concat(( train_df, X), axis=1)
 X_test = svd.transform(X_test)
 X_test = pd.DataFrame(X_test, columns=['svd1_{}'.format(i)for i in range(n_components)])
 test_df = pd.concat(( test_df.reset_index(drop = True), X_test), axis=1)
 print(test_df.shape )
train = base.copy()
test = baseTe.copy()
features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]
ordinal = [feat for feat in features if feat not in ohe] + ['day', 'month']
train = AntMapping(train, ordinal)
 test = AntMapping(test , ordinal)
 train, test, count_cols = CountEncoding(train, ordinal, test)
 ordinal += count_cols
scaler = RobustScaler(quantile_range=(10.0, 90.0))
oof10, pred10, score['Ant-CE'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)
 print(f'AUC score for Max Imputation of ordinals with OHE on non-ordinal columns with CountEncoding of Ordinals: {score["Ant-CE"]}' )<categorify>
train = base.copy()
test = baseTe.copy()
features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2']]
ordinal = [feat for feat in features if feat not in ohe]
train[features] = SiavashMapping(train[features])
 test[features] = SiavashMapping(test[features])
 imp = SimpleImputer(strategy='median')
 train[ordinal] = imp.fit_transform(train[ordinal])
 test[ordinal] = imp.transform(test[ordinal])
 train[ordinal] = train[ordinal].astype(np.int16)
 test[ordinal] = test[ordinal].astype(np.int16)
 scaler = RobustScaler(quantile_range=(10.0, 90.0))
oof11, pred11, score['median-OrdPartial'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)
 print(f'AUC score for median Imputation of ord_0, ord_1 and ord_2 columns: {score["median-OrdPartial"]}' )<categorify>
split_char = '/'
train_df_ids = train_df[['PetID']]
test_df_ids = test_df[['PetID']]
train_df_imgs = pd.DataFrame(train_image_files)
 train_df_imgs.columns = ['image_filename']
train_imgs_pets = train_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 def getSize(filename):
st = os.stat(filename)
 return st.st_size
def getDimensions(filename):
img_size = Image.open(filename ).size
return img_size
train_df_imgs['image_size'] = train_df_imgs['image_filename'].apply(getSize)
 train_df_imgs['temp_size'] = train_df_imgs['image_filename'].apply(getDimensions)
 train_df_imgs['width'] = train_df_imgs['temp_size'].apply(lambda x : x[0])
 train_df_imgs['height'] = train_df_imgs['temp_size'].apply(lambda x : x[1])
 train_df_imgs = train_df_imgs.drop(['temp_size'], axis=1)
 test_df_imgs['image_size'] = test_df_imgs['image_filename'].apply(getSize)
 test_df_imgs['temp_size'] = test_df_imgs['image_filename'].apply(getDimensions)
 test_df_imgs['width'] = test_df_imgs['temp_size'].apply(lambda x : x[0])
 test_df_imgs['height'] = test_df_imgs['temp_size'].apply(lambda x : x[1])
 test_df_imgs = test_df_imgs.drop(['temp_size'], axis=1)
 aggs = {
'image_size': ['sum', 'mean', 'var'],
'width': ['sum', 'mean', 'var'],
'height': ['sum', 'mean', 'var'],
}
agg_train_imgs = train_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()
agg_test_imgs = test_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index()
train_df = train_df.merge(agg_train_imgs, how='left', on='PetID')
 test_df = test_df.merge(agg_test_imgs, how='left', on='PetID' )
train = base.copy()
test = baseTe.copy()
features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]
ordinal = [feat for feat in features if feat not in ohe]
train[features] = SiavashMapping(train[features])
 test[features] = SiavashMapping(test[features])
 imp = SingleImputer(strategy='norm')
 train[ordinal] = imp.fit_transform(train[ordinal])
 test[ordinal] = imp.transform(test[ordinal])
 train[ordinal] = train[ordinal].astype(np.int16)
 test[ordinal] = test[ordinal].astype(np.int16)
 scaler = RobustScaler(quantile_range=(10.0, 90.0))
oof12, pred12, score['Norm'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)
 print(f'AUC score for Norm Imputation of ordinals: {score["Norm"]}' )<categorify>
print('scaling num_cols')
 for col in num_cols:
print('scaling {}'.format(col))
try:
col_mean = train_df[col].mean()
train_df[col].fillna(col_mean, inplace=True)
 test_df[col].fillna(col_mean, inplace=True)
 scaler = StandardScaler()
train_df[col] = scaler.fit_transform(train_df[col].values.reshape(-1, 1))
test_df[col] = scaler.transform(test_df[col].values.reshape(-1, 1))
except:
print("*********" )
train = base.copy()
test = baseTe.copy()
features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]
ordinal = [feat for feat in features if feat not in ohe]
train[features] = SiavashMapping(train[features])
 test[features] = SiavashMapping(test[features])
 imp = SingleImputer(strategy='locf')
 train[ordinal] = imp.fit_transform(train[ordinal])
 test[ordinal] = imp.transform(test[ordinal])
 train[ordinal] = train[ordinal].astype(np.int16)
 test[ordinal] = test[ordinal].astype(np.int16)
 scaler = RobustScaler(quantile_range=(10.0, 90.0))
oof13, pred13, score['LOCF'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)
 print(f'AUC score for LOCF Imputation of ordinals: {score["LOCF"]}' )<categorify>
def rmse(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true)))
 def get_input_features(df):
nn_nums= [x for x in list(num_cols)
 ]
X = {'description':pad_sequences(df['Description'], maxlen=maxlen)}
X['numerical'] = np.array(df[nn_nums])
 X['bow_inputs'] = np.array(df[list(["svd1_" + str(i)for i in range(n_components)] +
["SVD_metadata_annots_top_desc_" + str(i)for i in range(16)] +
["SVD_sentiment_entities_" + str(i)for i in range(16)] 
)])
 X['img_inputs'] = np.array(df[list(["img_svd1_" + str(i)for i in range(n_components)] + ["img_svd2_" + str(i)for i in range(n_components)] 
)])
 for cat in cat_cols:
X[cat] = np.array(df[cat])
 return X
train = base.copy()
test = baseTe.copy()
features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]
ordinal = [feat for feat in features if feat not in ohe]
train[features] = SiavashMapping(train[features])
 test[features] = SiavashMapping(test[features])
 imp = SingleImputer(strategy='default univariate')
 train[ordinal] = imp.fit_transform(train[ordinal])
 test[ordinal] = imp.transform(test[ordinal])
 train[ordinal] = train[ordinal].astype(np.int16)
 test[ordinal] = test[ordinal].astype(np.int16)
 scaler = RobustScaler(quantile_range=(10.0, 90.0))
oof14, pred14, score['DefUnivariate'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)
 print(f'AUC score for Default Univariate Imputation: {score["DefUnivariate"]}' )<categorify>
def kappa_loss(y_pred, y_true, y_pow=2, eps=1e-10, N=5, bsize=256, name='kappa'):

 with tf.name_scope(name):
y_true = tf.to_float(y_true)
 repeat_op = tf.to_float(tf.tile(tf.reshape(tf.range(0, N), [N, 1]), [1, N]))
repeat_op_sq = tf.square(( repeat_op - tf.transpose(repeat_op)))
 weights = repeat_op_sq / tf.to_float(( N - 1)** 2)
 pred_ = y_pred ** y_pow
try:
pred_norm = pred_ /(eps + tf.reshape(tf.reduce_sum(pred_, 1), [-1, 1]))
except Exception:
pred_norm = pred_ /(eps + tf.reshape(tf.reduce_sum(pred_, 1), [bsize, 1]))
hist_rater_a = tf.reduce_sum(pred_norm, 0)
 hist_rater_b = tf.reduce_sum(y_true, 0)
 conf_mat = tf.matmul(tf.transpose(pred_norm), y_true)
 nom = tf.reduce_sum(weights * conf_mat)
 denom = tf.reduce_sum(weights * tf.matmul(
 tf.reshape(hist_rater_a, [N, 1]), tf.reshape(hist_rater_b, [1, N])) /
tf.to_float(bsize))
return nom /(denom + eps )
train = base.copy()
test = baseTe.copy()
features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = features
ordinal = None
train[features] = SiavashMapping(train[features])
 test[features] = SiavashMapping(test[features])
 imp = SingleImputer(strategy='interpolate')
 train[features] = imp.fit_transform(train[features])
 test[features] = imp.transform(test[features])
 train[features] = train[features].astype(np.int16)
 test[features] = test[features].astype(np.int16)
 scaler = RobustScaler(quantile_range=(10.0, 90.0))
oof15, pred15, score['Interpolate-OHE'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)
 print(f'Average AUC score for Interpolate Imputation with OHE om all columns: {score["Interpolate-OHE"]}' )<categorify>
def make_model(softmax = False):
k.clear_session()
categorical_inputs = []
for cat in cat_cols:
categorical_inputs.append(Input(shape=[1], name=cat))
categorical_embeddings = []
for i, cat in enumerate(cat_cols):
categorical_embeddings.append(
 Embedding(embed_sizes[i], 10, name = cat + "_embed" )(categorical_inputs[i]))
categorical_logits = Concatenate()([Flatten()(cat_emb)for cat_emb in categorical_embeddings])
 categorical_logits = Dropout (.5 )(categorical_logits)
 categorical_logits = Dense(50, activation = 'relu' )(categorical_logits)
 numerical_inputs = Input(shape=[X_train["numerical"].shape[1]], name = 'numerical')
 numerical_logits = Dropout (.2 )(numerical_inputs)
 numerical_logits = Dense(50, activation = 'relu' )(numerical_logits)
 numerical_logits = Dense(50, activation = 'relu' )(numerical_logits)
 img_inputs = Input(shape = [n_components * 2], name = "img_inputs")
 img_logits = Dropout (.2 )(img_inputs)
 bow_inputs = Input(shape = [n_components * 2], name = "bow_inputs")
 bow_logits = Dropout (.2 )(bow_inputs)
 text_inp = Input(shape=[maxlen], name='description')
 text_embed = Embedding(nb_words, embed_size, weights=[embedding_matrix], trainable=False )(text_inp)
 emb_desc = SpatialDropout1D (.6 )(text_embed)
 filter_sizes=[1,2,3,4]
convs = []
for filter_size in filter_sizes:
conv = Conv1D(8, kernel_size=(filter_size),
kernel_initializer="normal", activation="relu" )(emb_desc)
 convs.append(MaxPool1D(pool_size=(maxlen-filter_size+1))(conv))
text_logits = concatenate(convs)
 avg_pool = GlobalAveragePooling1D()(text_logits)
 max_pool = GlobalMaxPooling1D()(text_logits)
 text_logits = Concatenate()([avg_pool, max_pool])
 x = Concatenate()([
bow_logits,
categorical_logits,
text_logits,
numerical_logits,
img_logits
])
 x = Dense(300, activation = 'relu' )(x)
 x = Dropout (.2 )(x)
 x = Dense(200, activation = 'relu' )(x)
 x = Dropout (.2 )(x)
 x = Dense(100, activation = 'relu' )(x)
 if softmax == True:
out = Dense(5, activation = 'softmax' )(x)
 else:
out = Dense(1, activation = 'sigmoid' )(x)
 model = Model(inputs=[text_inp] + categorical_inputs + [numerical_inputs] + [bow_inputs] + [img_inputs],outputs=out)
 if softmax == True:
loss = kappa_loss
else:
loss = root_mean_squared_error
model.compile(optimizer=Adam(lr = 0.0005), loss = loss)
 return model
train = base.copy()
test = baseTe.copy()
features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = features
ordinal = None
train[features] = SiavashMapping(train[features])
 test[features] = SiavashMapping(test[features])
 imp = SoftImpute(max_iters=100, verbose=False)
 train[features] = imp.fit_transform(train[features])
 test[features] = imp.fit_transform(test[features])
 train[features] = train[features].astype(np.int16)
 test[features] = test[features].astype(np.int16)
 scaler = RobustScaler(quantile_range=(10.0, 90.0))
oof16, pred16, score['SoftImpute-OHE'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)
 print(f'AUC score for SoftImpute Imputation : {score["SoftImpute-OHE"]}' )<categorify>
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )
train = base.copy()
test = baseTe.copy()
features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2']]
ordinal = [feat for feat in features if feat not in ohe]
train[features] = SiavashMapping(train[features])
 test[features] = SiavashMapping(test[features])
 imp = SingleImputer(strategy='least squares')
 train[ordinal] = imp.fit_transform(train[ordinal])
 test[ordinal] = imp.transform(test[ordinal])
 train[ordinal] = train[ordinal].astype(np.int16)
 test[ordinal] = test[ordinal].astype(np.int16)
 scaler = RobustScaler(quantile_range=(10.0, 90.0))
oof17, pred17, score['LQ-PartialOHE'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)
 print(f'AUC score for LQ Imputation with Partial OHE : {score["LQ-PartialOHE"]}' )<categorify>
def to_bins(x, borders):
for i in range(len(borders)) :
if x <= borders[i]:
return i
return len(borders)
 class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _loss(self, coef, X, y, idx):
X_p = np.array([to_bins(pred, coef)for pred in X])
 ll = -quadratic_weighted_kappa(y, X_p)
 return ll
def fit(self, X, y):
coef = [1.5, 2.0, 2.5, 3.0]
golden1 = 0.618
golden2 = 1 - golden1
ab_start = [(1, 2),(1.5, 2.5),(2, 3),(2.5, 3.5)]
for it1 in range(10):
for idx in range(4):
a, b = ab_start[idx]
coef[idx] = a
la = self._loss(coef, X, y, idx)
 coef[idx] = b
lb = self._loss(coef, X, y, idx)
 for it in range(20):
if la > lb:
a = b -(b - a)* golden1
coef[idx] = a
la = self._loss(coef, X, y, idx)
 else:
b = b -(b - a)* golden2
coef[idx] = b
lb = self._loss(coef, X, y, idx)
 self.coef_ = {'x': coef}
def predict(self, X, coef):
X_p = np.array([to_bins(pred, coef)for pred in X])
 return X_p
def coefficients(self):
return self.coef_['x']
train = base.copy()
test = baseTe.copy()
features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]
ordinal = [feat for feat in features if feat not in ohe]
train[features] = SiavashMapping(train[features])
 test[features] = SiavashMapping(test[features])
 imp = SingleImputer(strategy='stochastic')
 train[ordinal] = imp.fit_transform(train[ordinal])
 test[ordinal] = imp.transform(test[ordinal])
 train[ordinal] = train[ordinal].astype(np.int16)
 test[ordinal] = test[ordinal].astype(np.int16)
 scaler = RobustScaler(quantile_range=(10.0, 90.0))
oof18, pred18, score['stochastic'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)
 print(f'AUC score for stochastic Imputation : {score["stochastic"]}' )<categorify>
rescuerID = pd.read_csv(".. /input/petfinder-adoption-prediction/train/train.csv")["RescuerID"]
train = base.copy()
test = baseTe.copy()
features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]
ordinal = [feat for feat in features if feat not in ohe]
train[features] = SiavashMapping(train[features])
 test[features] = SiavashMapping(test[features])
 imp = IterativeImputer(max_iter=500, initial_strategy='most_frequent', random_state=SEED)
 train[features] = imp.fit_transform(train[features])
 test[features] = imp.transform(test[features])
 train[features] = train[features].astype(np.int16)
 test[features] = test[features].astype(np.int16)
 scaler = RobustScaler(quantile_range=(10.0, 90.0))
oof19, pred19, score['Iterative'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)
 print(f'AUC score for Iterative Imputation : {score["Iterative"]}' )<categorify>
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true)) )
train = base.copy()
test = baseTe.copy()
drop_idx = base[(base.isna().sum(axis=1)>3)].index.values
drop_idx = [i for i in drop_idx]
features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]
ordinal = [feat for feat in features if feat not in ohe]
train[features] = SiavashMapping(train[features])
 test[features] = SiavashMapping(test[features])
 imp = IterativeImputer(max_iter=500, initial_strategy='most_frequent', random_state=SEED)
 train[features] = imp.fit_transform(train[features])
 test[features] = imp.transform(test[features])
 train[features] = train[features].astype(np.int16)
 test[features] = test[features].astype(np.int16)
 scaler = RobustScaler(quantile_range=(10.0, 90.0))
oof20, pred20, score['IterativeWithDrop'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=drop_idx)
 print(f'AUC score for Iterative Imputation With threshold Drop : {score["IterativeWithDrop"]}' )<categorify>
num_folds = 10
softmax = False
kf = GroupKFold(n_splits=num_folds)
 y = train_df["AdoptionSpeed"]
fold_splits = kf.split(train_df, y, rescuerID)
 oof = np.zeros(( y.shape))
test_preds = np.zeros(( test_df.shape[0]))
if softmax == True:
test_preds = np.zeros(( test_df.shape[0], 5))
oof = np.zeros(shape =(y.shape[0], 5))
for i,(dev_index, val_index)in enumerate(fold_splits):
tr_df = train_df.iloc[dev_index, :]
val_df = train_df.iloc[val_index, :]
X_train = get_input_features(tr_df)
 X_valid = get_input_features(val_df)
 X_test = get_input_features(test_df)
 model = make_model(softmax = softmax)
 ckpt = ModelCheckpoint("model" + str(i)+ ".h5", monitor='val_loss', save_best_only = True, verbose = False)
 rlr = ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=5, mode='auto', verbose = False)
 if softmax == True:
y_train = np.zeros(shape =(tr_df.shape[0], 5))
y_valid = np.zeros(shape =(val_df.shape[0], 5))
for j, l in enumerate(tr_df['AdoptionSpeed'].values):
j = int(j)
 l = int(l)
 y_train[j,l] = 1
for j, l in enumerate(val_df['AdoptionSpeed'].values):
j = int(j)
 l = int(l)
 y_valid[j,l] = 1
else:
y_train = tr_df['AdoptionSpeed'].values / 4
y_valid = val_df['AdoptionSpeed'].values / 4
hist = model.fit(X_train, y_train, validation_data =(X_valid,y_valid), batch_size = 500, epochs = 35, verbose = 2, callbacks = [ckpt, rlr])
 model.load_weights("model" + str(i)+ ".h5")
 if softmax == True:
val_preds = model.predict(X_valid)
 oof[val_index] = val_preds
test_pred = model.predict(X_test)
 test_preds +=(test_pred/num_folds)
 else:
val_preds = model.predict(X_valid)[:, 0] * 4
y_valid = y_valid * 4
optR = OptimizedRounder()
optR.fit(val_preds, y_valid)
 coefficients = optR.coefficients()
val_pred_rounded = optR.predict(val_preds, coefficients)
 oof[val_index] = val_preds
test_pred = model.predict(X_test)[:, 0] * 4
test_preds +=(test_pred/num_folds)
 if softmax == True:
oof_rounded = np.argmax(oof, axis = 1)
 test_rounded = np.argmax(test_preds, axis = 1)
 else:
optR = OptimizedRounder()
optR.fit(oof, y)
 coefficients = optR.coefficients()
oof_rounded = optR.predict(oof, coefficients)
 test_rounded = optR.predict(test_preds, coefficients )
train = base.copy()
test = baseTe.copy()
drop_idx = base[(base.isna().sum(axis=1)>3)].index.values
drop_idx = [i for i in drop_idx]
features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5']]
ordinal = [feat for feat in features if feat not in ohe]
train[features] = SiavashMapping(train[features])
 test[features] = SiavashMapping(test[features])
 imp = IterativeImputer(max_iter=500, initial_strategy='most_frequent', random_state=SEED, add_indicator=True)
 indicator_cols = [feat+'_ind' for feat in ordinal]
for col in indicator_cols:
train[col] = 0
test[col] = 0
train[col] = train[col].astype(np.uint8)
 test[col] = test[col].astype(np.uint8)
 train[ordinal+indicator_cols] = imp.fit_transform(train[ordinal])
 test[ordinal+indicator_cols] = imp.transform(test[ordinal])
 train[ordinal] = train[ordinal].astype(np.int16)
 test[ordinal] = test[ordinal].astype(np.int16)
 scaler = RobustScaler(quantile_range=(10.0, 90.0))
ohe += indicator_cols
oof21, pred21, score['IterativeWithIndicator'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=drop_idx)
 print(f'AUC score for Iterative Imputation With Indicator : {score["IterativeWithIndicator"]}' )<categorify>
print(num_cols)
 print(quadratic_weighted_kappa(y, oof_rounded))
oof_nn = oof
test_preds_nn = test_preds
train = base.copy()
test = baseTe.copy()
train['nulls'] = train.isna().sum(axis=1)
 test['nulls'] = test.isna().sum(axis=1)
 features = [feat for feat in train.columns if feat not in ['target','id']]
ohe = [feat for feat in features if feat not in ['ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5', 'nulls']]
ordinal = [feat for feat in features if feat not in ohe]
train[features] = SiavashMapping(train[features])
 test[features] = SiavashMapping(test[features])
 imp = IterativeImputer(max_iter=500, initial_strategy='most_frequent', random_state=SEED)
 train[ordinal] = imp.fit_transform(train[ordinal])
 test[ordinal] = imp.transform(test[ordinal])
 train[ordinal] = train[ordinal].astype(np.int16)
 test[ordinal] = test[ordinal].astype(np.int16)
 scaler = RobustScaler(quantile_range=(10.0, 90.0))
oof22, pred22, score['IterativeWithSum'] = LogRegClf(train=train, test=test, ordinal=ordinal, ohe=ohe, scaler=scaler, seed=SEED, splits=SPLITS, drop_idx=None)
 print(f'AUC score for Iterative Imputation with sum of missing values: {score["IterativeWithSum"]}' )<create_dataframe>
X_temp = deepcopy(pd.concat([train_df, test_df], axis = 0))
scores = pd.DataFrame(score, index=['OOF-AUC'] ).T.sort_values(by='OOF-AUC', ascending=False )<prepare_x_and_y>
n_bins = 5
for c in num_cols:
if X_temp[c].nunique() < 5:
continue
vals = X_temp[c].values.reshape(-1, 1)
 est = KBinsDiscretizer(n_bins=n_bins, encode='ordinal')
 est.fit(vals)
 vals = est.transform(vals ).reshape(1, -1)[0]
X_temp[c] = vals
def StackModels() :
idx = baseTe.id.values
train_oofs = [oof1, oof2, oof3, oof4, oof5, oof6, oof7, oof8, oof9, oof10, oof11, oof12, oof13, oof14, oof15, oof16, oof17, oof18, oof19]
test_oofs = [pred1, pred2, pred3, pred4, pred5, pred6, pred7, pred8, pred9, pred10, pred11, pred12, pred13, pred14, pred15, pred16, pred17, pred18, pred19]
X_train = pd.concat([pd.DataFrame(file)for file in train_oofs], axis=1)
 X_test = pd.concat([pd.DataFrame(file)for file in test_oofs], axis=1)
 X_train.columns = ['y_' + str(i)for i in range(len(train_oofs)) ]
X_test.columns = ['y_' + str(i)for i in range(len(train_oofs)) ]
X_train = pd.concat([X_train, base[['target']]], axis=1)
 X_test = pd.concat([X_test, baseTe[['target']]], axis=1)
 for i,(oof, pred)in enumerate(zip(train_oofs, test_oofs)) :
train_oofs[i] = rankdata(oof)/len(oof)
 test_oofs[i] = rankdata(pred)/len(pred)
 for f in X_train.columns:
X_train[f] = X_train[f].astype('float32')
 X_test[f] = X_test[f].astype('float32')
 features = np.array([f for f in X_train.columns if f not in ['target']])
 target = ['target']
oof_pred_final = np.zeros(( len(base),))
y_pred_final = np.zeros(( len(baseTe),))
skf = StratifiedKFold(n_splits=SPLITS, shuffle=True, random_state=SEED)
 model = RidgeClassifier()
selector = RFECV(model, step=1, cv = skf, scoring='roc_auc', verbose=0, n_jobs=4)
 selector.fit(X_train[features], X_train[target])
 selected_features = [i for i, y in enumerate(selector.ranking_)if y == 1]
selected_features = features[selected_features]
for fold,(tr_ind, val_ind)in enumerate(skf.split(X_train, X_train[target])) :
x_tr, x_val = X_train[selected_features].iloc[tr_ind], X_train[selected_features].iloc[val_ind]
y_tr, y_val = X_train[target].iloc[tr_ind], X_train[target].iloc[val_ind]
train_set = {'X':x_tr, 'y':y_tr}
val_set = {'X':x_val, 'y':y_val}
model = RidgeClassifier()
model.fit(train_set['X'],train_set['y'])
 fold_pred = model.decision_function(val_set['X'])
 oof_pred_final[val_ind] = fold_pred
y_pred_final += model.decision_function(X_test[selected_features])/(SPLITS)
 oof_auc_score = roc_auc_score(base[target], oof_pred_final)
 print(f'OOF Stack ROC-AUC Score is: {oof_auc_score:.7f}')
 y_pred_final = rankdata(y_pred_final)/len(y_pred_final)
 np.save('oof_pred_final.npy',oof_pred_final)
 np.save('y_pred_final.npy', y_pred_final)
 print('*'* 36)
 print(' OOF files saved!')
 print('*'* 36)
 submission = pd.DataFrame.from_dict({
'id': idx,
'target': y_pred_final
})
 submission.to_csv('submission.csv', index=False)
 print('*'* 36)
 print(' Submission file saved!')
 print('*'* 36)
 return 
<load_from_csv>
train_df_ffm = X_temp.iloc[:train_df.shape[0], :]
test_df_ffm = X_temp.iloc[train_df.shape[0]:, :]
df_train = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')
 df_test = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv')
 df_sub = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/sample_submission.csv' )<sort_values>
field_features = defaultdict()
global max_val
max_val = 1
global max_val
line = []
label = []
dtypes = {}
for feature in num_cols:
dtypes[feature] = "float16"
for feature in cat_cols:
dtypes[feature] = "category"
dont_use = ['RescuerID', 'AdoptionSpeed', 'Description', 'PetID']
too_many_vals = [""]
categories = [k for k, v in dtypes.items() if k not in dont_use]
categories_index = dict(zip(categories, range(len(categories))))
def generate_ffm_file(index, train = True):
global max_val
ffeatures = []
if train == True:
path = 'train.libffm'
else:
path = 'valid.libffm'
with open(path, 'w')as the_file:
for t,(index, row)in enumerate(train_df_ffm.iloc[index, :].iterrows()):
if t % 5000 == 0:
print(t, len(field_features), max_val)
 label = [str(int(row['AdoptionSpeed'])) ]
ffeatures = []
for field in categories:
if field == 'AdoptionSpeed':
continue
feature = row[field]
if feature == '':
feature = "unk"
if field not in num_cols:
ff = field + '_____' + str(feature)
 else:
if feature == "unk" or float(feature)== -1:
ff = field + '_____' + str(0)
 else:
if field in too_many_vals:
ff = field + '_____' + str(int(round(math.log(1 + float(feature)))))
 else:
ff = field + '_____' + str(feature)
 if ff not in field_features:
if len(field_features)== 0:
field_features[ff] = 1
max_val += 1
else:
field_features[ff] = max_val + 1
max_val += 1
fnum = field_features[ff]
ffeatures.append('{}:{}:1'.format(categories_index[field], str(fnum)))
 line = label + ffeatures
the_file.write('{}
'.format(' '.join(line)) )
df_train.sort_index(inplace=True )<prepare_x_and_y>
optR = OptimizedRounder()
optR.fit(ffm_oof_train, y)
 coefficients = optR.coefficients()
train_ffm_rounded = optR.predict(ffm_oof_train, coefficients)
 print(quadratic_weighted_kappa(train_ffm_rounded, y))
y_train = df_train['target']
test_id = df_test['id']
df_train.drop(['target', 'id'], axis=1, inplace=True)
 df_test.drop('id', axis=1, inplace=True )<categorify>
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))
cat_feat_to_encode = df_train.columns.tolist()
smoothing=0.20
oof = pd.DataFrame([])
 for tr_idx, oof_idx in StratifiedKFold(n_splits=5, random_state= 1024, shuffle=True ).split(df_train, y_train):
ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(df_train.iloc[tr_idx, :], y_train.iloc[tr_idx])
 oof = oof.append(ce_target_encoder.transform(df_train.iloc[oof_idx, :]), ignore_index=False)
 ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(df_train, y_train)
 df_train = oof.sort_index()
df_test = ce_target_encoder.transform(df_test )<prepare_x_and_y>
train = deepcopy(train_df)
 test = deepcopy(test_df )
x_train = df_train.iloc[:,:].values
x_test = df_test.iloc[:,:].values<choose_model_class>
train.loc[:, cat_cols] = train[cat_cols].astype('category')
 test.loc[:, cat_cols] = test[cat_cols].astype('category')
 print(train.shape)
 print(test.shape)
 train.head()
classifier = lgb.LGBMClassifier(**{
'learning_rate': 0.05,
'feature_fraction': 0.1,
'min_data_in_leaf' : 12,
'max_depth': 3,
'reg_alpha': 1,
'reg_lambda': 1,
'objective': 'binary',
'metric': 'auc',
'n_jobs': -1,
'n_estimators' : 5000,
'feature_fraction_seed': 42,
'bagging_seed': 42,
'boosting_type': 'gbdt',
'verbose': 1,
'is_unbalance': True,
'boost_from_average': False} )<train_model>
train.drop(set(['Name', 'RescuerID', 'Description', 'Description1', 'PetID', 'AdoptionSpeed']), axis=1, inplace=True)
 test.drop(set(['Name', 'RescuerID', 'Description', 'Description1', 'PetID']), axis=1, inplace=True )
classifier.fit(x_train, y_train )<predict_on_test>
def run_cv_model(train, test, target, model_fn, params={}, eval_fn=None, label='model', num_iters = 1, num_folds = 19,):
cv_scores = []
qwk_scores = []
pred_full_test = 0
pred_train = np.zeros(( train.shape[0], num_folds))
all_coefficients = np.zeros(( num_folds * num_iters, 4))
feature_importance_df = pd.DataFrame()
i = 1
for j in range(num_iters):
kf = GroupKFold(n_splits=num_folds)
 fold_splits = kf.split(train, target, rescuerID)
 for dev_index, val_index in fold_splits:
print('Started ' + label + ' fold ' + str(i)+ '/5')
 if isinstance(train, pd.DataFrame):
dev_X, val_X = train.iloc[dev_index], train.iloc[val_index]
dev_y, val_y = target[dev_index], target[val_index]
else:
dev_X, val_X = train[dev_index], train[val_index]
dev_y, val_y = target[dev_index], target[val_index]
params2 = params.copy()
pred_val_y, pred_test_y, importances, coefficients, qwk = model_fn(dev_X, dev_y, val_X, val_y, test, params2)
 pred_full_test +=(pred_test_y / num_iters)
 pred_train[val_index] +=(pred_val_y/num_iters)
 all_coefficients[i-1, :] = coefficients
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 qwk_scores.append(qwk)
 print(label + ' cv score {}: RMSE {} QWK {}'.format(i, cv_score, qwk))
fold_importance_df = pd.DataFrame()
fold_importance_df['feature'] = train.columns.values
fold_importance_df['importance'] = importances
fold_importance_df['fold'] = i
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 i += 1
pred_full_test =(pred_full_test / num_folds)
 print('{} cv RMSE scores : {}'.format(label, cv_scores))
print('{} cv mean RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv std RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv QWK scores : {}'.format(label, qwk_scores))
print('{} cv mean QWK score : {}'.format(label, np.mean(qwk_scores)))
 print('{} cv std QWK score : {}'.format(label, np.std(qwk_scores)))
 results = {'label': label,
'train': pred_train, 'test': pred_full_test,
'cv': cv_scores, 'qwk': qwk_scores,
'importance': feature_importance_df,
'coefficients': all_coefficients}
return results
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 70,
'max_depth': 7,
'learning_rate': 0.02,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.02,
'min_child_samples': 150,
'min_child_weight': 0.2,
'lambda_l2': 0.05,
'verbosity': -1,
'data_random_seed': 24,
'early_stop': 100,
'verbose_eval': 100,
'num_rounds': 10000}
def runLGB(train_X, train_y, test_X, test_y, test_X2, params):
print('Prep LGB')
 d_train = lgb.Dataset(train_X, label=train_y)
 d_valid = lgb.Dataset(test_X, label=test_y)
 watchlist = [d_train, d_valid]
print('Train LGB')
 num_rounds = params.pop('num_rounds')
 verbose_eval = params.pop('verbose_eval')
 early_stop = None
if params.get('early_stop'):
early_stop = params.pop('early_stop')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 print('Predict 1/2')
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 optR = OptimizedRounder()
optR.fit(pred_test_y, test_y)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(pred_test_y, coefficients)
 print("Valid Counts = ", Counter(test_y))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(test_y, pred_test_y_k)
 print("QWK = ", qwk)
 print('Predict 2/2')
 pred_test_y2 = model.predict(test_X2, num_iteration=model.best_iteration)
 return pred_test_y.reshape(-1, 1), pred_test_y2.reshape(-1, 1), model.feature_importance("gain"), coefficients, qwk
y_pred = classifier.predict_proba(x_test)[:,1]<save_to_csv>
no_drop = ['Color1', 'Color2', 'Color3', 'Dewormed', 'Fee', 'FurLength',
'Gender', 'Health', 'MaturitySize','PhotoAmt', 'Quantity','Sterilized', 'Type', 'Vaccinated',
'state_gdp', 'state_population',]
df_sub = pd.DataFrame()
df_sub['id'] = test_id
df_sub['target'] = y_pred
df_sub.to_csv('submission.csv', index=False )<set_options>
keep_features =(['Age', 'Breed1', 'Breed2', 'Color1', 'Color2', 'Color3', 'Fee', 'Quantity',
'MaturitySize','FurLength','Vaccinated','Dewormed', 'Sterilized','Health','Quantity','Fee',
'VideoAmt','PhotoAmt', 'state_gdp','state_population', 'doc_sent_mag', 'doc_sent_score', 'vertex_x', 'vertex_y',
'bounding_confidence', 'bounding_importance','dominant_blue', 'dominant_green', 'dominant_red', 'dominant_pixel_frac',
'dominant_score', 'label_description', 'label_score',
'RescuerID_COUNT','State', 'Sterilized', 'agg_mean_Age', 'agg_mean_Color1',
'agg_mean_Color2', 'agg_mean_Color3', 'agg_mean_Dewormed',
'agg_mean_Fee', 'agg_mean_FurLength', 'agg_mean_Gender',
'agg_mean_PhotoAmt', 'agg_mean_Quantity', 'agg_mean_Sterilized',
'agg_mean_Vaccinated', 'agg_mean_VideoAmt', 'agg_std_Age',
'agg_std_Color1', 'agg_std_Color2', 'agg_std_Color3',
'agg_std_Dewormed', 'agg_std_Fee', 'agg_std_FurLength',
'agg_std_Gender', 'agg_std_PhotoAmt', 'agg_std_Quantity',
'agg_std_Sterilized', 'agg_std_Vaccinated', 'label_score',
'metadata_metadata_annots_score_MAX',
'metadata_metadata_annots_score_MEAN',
'metadata_metadata_annots_score_MIN',
'metadata_metadata_annots_score_SUM',
'metadata_metadata_annots_score_VAR',
'metadata_metadata_color_pixelfrac_MIN',
'metadata_metadata_color_pixelfrac_SUM',
'metadata_metadata_color_pixelfrac_VAR',
'metadata_metadata_color_score_SUM',
'metadata_metadata_color_score_VAR', 'vertex_x', 'vertex_y','RescuerID_UNIQUE']
+["svd1_" + str(i)for i in range(n_components)]
+["img_svd1_" + str(i)for i in range(n_components)]
+["img_svd2_" + str(i)for i in range(n_components)] 
)
warnings.filterwarnings('ignore')
 plt.style.use('ggplot')
<load_from_csv>
results = run_cv_model(train[keep_features], test[keep_features], y, runLGB, params, rmse, 'lgb' )
train = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')
 test = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv')
 sample_submission = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/sample_submission.csv' )<concatenate>
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'eta': 0.0123,
'subsample': 1.0,
'colsample_bytree': 0.6,
'min_child_weight': 1.0,
'gamma' :.5,
'max_depth' : 5,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
}
all_df = pd.concat([train, test], axis=0, ignore_index=True )<count_unique_values>
def run_xgb(params, X_train, X_test):
n_splits = 20
verbose_eval = 1000
num_rounds = 30000
early_stop = 500
kf = GroupKFold(n_splits=n_splits)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, y, rescuerID):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = y.iloc[train_idx]
y_val = y.iloc[valid_idx]
d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test
all_df.index.nunique() , len(all_df.index )<drop_column>
final_feats = keep_features
all_df = all_df.drop('id', axis=1 )<count_unique_values>
X_temp = pd.concat([train, test], ignore_index=True, sort=False)
 for i in cat_cols:
X_temp = pd.concat([pd.get_dummies(X_temp.loc[:, i], prefix = str(i)) , X_temp], axis = 1)
 final_feats = final_feats + [i + "_"+ str(x)for x in range(X_temp.loc[:, i].nunique())]
train = X_temp.loc[:train.shape[0]-1, :]
test = X_temp.loc[train.shape[0]:, :]
nunique_vals = list()
for column in all_df:
nunique_vals.append(all_df[column].nunique())
 pd.DataFrame({'columns': all_df.columns,
'num_of_unique': nunique_vals} )<count_unique_values>
train[cat_cols] = train[cat_cols].astype(int)
 test[cat_cols] = test[cat_cols].astype(int )
for column in all_df.columns:
unique_values = all_df[column].unique()
print(f'Statistics fot column: {column}')
 print(f'Column unique values:
{unique_values}')
 print(f'Number of unique values: {len(unique_values)}')
 print(f'Number of NAN values: {all_df[column].isna().sum() }')
 print('_' * 50 )<create_dataframe>
optR = OptimizedRounder()
optR.fit(oof_train, y)
 coefficients = optR.coefficients()
oof_xgb = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(y, oof_xgb)
 print("QWK = ", qwk)
month_temperature = all_df.groupby(['month', 'ord_2'])['ord_2'].count().to_frame()
month_temperature = month_temperature.rename(columns={'ord_2': 'num_of_days'})
 month_temperature = month_temperature.reset_index()<drop_column>
test_preds_xgb = optR.predict(oof_test.mean(axis=1), coefficients ).astype(np.int8 )
all_df['ord_5_1'] = all_df['ord_5'].str[0]
all_df['ord_5_2'] = all_df['ord_5'].str[1]
all_df = all_df.drop('ord_5', axis=1 )<feature_engineering>
optR = OptimizedRounder()
coefficients_ = np.mean(results['coefficients'], axis=0)
 print(coefficients_)
 train_predictions = [r[0] for r in results['train']]
oof_lgb = np.array(train_predictions)
 train_predictions = optR.predict(train_predictions, coefficients_ ).astype(int)
 Counter(train_predictions )
all_df['nan_features'] = all_df.isna().sum(axis=1 )<feature_engineering>
test_predictions = [r[0] for r in results['test']]
test_preds_lgb = np.array(test_predictions)
 test_predictions = optR.predict(test_predictions, coefficients_ ).astype(int )
all_df['month_sin'] = np.sin(( all_df['month'] - 1)*(2.0 * np.pi / 12))
all_df['month_cos'] = np.cos(( all_df['month'] - 1)*(2.0 * np.pi / 12))
all_df['day_sin'] = np.sin(( all_df['day'] - 1)*(2.0 * np.pi / 7))
all_df['day_cos'] = np.cos(( all_df['day'] - 1)*(2.0 * np.pi / 7))<define_variables>
optR = OptimizedRounder()
optR.fit(oof_lgb, y)
 coefficients = optR.coefficients()
print(coefficients)
 oof_rounded = optR.predict(oof_lgb, coefficients)
 print(quadratic_weighted_kappa(y, oof_rounded))
test_rounded_lgb = optR.predict(test_preds_lgb, coefficients )
categorical = ['bin_0', 'bin_1', 'bin_2', 'bin_3',
'bin_4', 'day', 'month', 'nom_0',
'nom_1', 'nom_2', 'nom_3', 'nom_4',
'nom_5', 'nom_6', 'nom_7', 'nom_8',
'nom_9', 'ord_0', 'ord_1', 'ord_2',
'ord_3', 'ord_4', 'ord_5_1', 'ord_5_2']
nom_5_9 = ['nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9']
continuous = ['month_sin', 'month_cos',
'day_sin', 'day_cos']<define_variables>
if softmax == False:
optR = OptimizedRounder()
optR.fit(oof_nn, y)
 coefficients = optR.coefficients()
print(coefficients)
 oof_rounded = optR.predict(oof_nn, coefficients)
 print(quadratic_weighted_kappa(y, oof_rounded))
test_rounded_nn = optR.predict(test_preds_nn, coefficients)
 if softmax == True:
oof_rounded = np.argmax(oof_nn, axis = 1)
 test_rounded_nn = np.argmax(test_preds_nn, axis = 1)
 print(quadratic_weighted_kappa(y, oof_rounded))
features = [x for x in all_df.columns
if x not in ['id', 'target'] + continuous]<categorify>
oof_models = [oof_nn, oof_lgb
, oof_train, ffm_oof_train
]
test_models = [test_preds_nn, test_preds_lgb
, oof_test.mean(axis = 1), ffm_oof_test
]
for feat in tqdm(features):
lbl_enc = preprocessing.LabelEncoder()
all_df[feat] = lbl_enc.fit_transform(all_df[feat].\
fillna('-1' ).\
astype(str ).values)
 all_df['target'] = all_df['target'].fillna(-1)
 all_df[continuous] = all_df[continuous].fillna(-2 )<statistical_test>
lr = Ridge(fit_intercept=False)
 lr.fit(np.array(oof_models ).T, y)
 print(lr.coef_)
 lr.coef_ = lr.coef_ * 1/(sum(lr.coef_))
print(lr.coef_)
 oof_lr = lr.predict(np.array(oof_models ).T)
 test_preds_lr = lr.predict(np.array(test_models ).T)
 optR = OptimizedRounder()
optR.fit(oof_lr, y)
 coefficients = optR.coefficients()
print(coefficients)
 oof_rounded = optR.predict(oof_lr, coefficients)
 print(quadratic_weighted_kappa(y, oof_rounded))
test_rounded_lr = optR.predict(test_preds_lr, coefficients )
def cramers_v(x, y):

 confusion_matrix = pd.crosstab(x, y)
 chi2 = chi2_contingency(confusion_matrix)[0]
n = confusion_matrix.sum().sum()
r, k = confusion_matrix.shape
phi_2 = chi2 / n
phi2corr = max(0, phi_2 -(( k - 1)*(r - 1)) /(n - 1))
rcorr = r -(( r - 1)** 2)/(n - 1)
 kcorr = k -(( k - 1)** 2)/(n - 1)
 res = np.sqrt(phi2corr / min(kcorr - 1, rcorr - 1))
return res
def conditional_entropy(x, y):

 y_counter = Counter(y)
 xy_counter = Counter(( list(zip(x, y))))
total_occurrences = sum(y_counter.values())
 entropy = 0.0
for xy in xy_counter.keys() :
p_xy = xy_counter[xy] / total_occurrences
p_y = y_counter[xy[1]] / total_occurrences
entropy += p_xy * np.log(p_y / p_xy)
 return entropy
def theils_u(x, y):

 s_xy = conditional_entropy(x, y)
 x_counter = Counter(x)
 total_occurrences = sum(x_counter.values())
 p_x = list(map(lambda n: n / total_occurrences, x_counter.values()))
s_x = entropy(p_x)
 if s_x == 0:
return 1
else:
return(s_x - s_xy)/ s_x<set_options>
submission_df = pd.read_csv(".. /input/petfinder-adoption-prediction/test/sample_submission.csv" )
plt.rcParams.update({'font.size': 10} )<define_variables>
submission_df["AdoptionSpeed"] = test_rounded_lr.astype(int)
 submission_df.to_csv("submission.csv", index=False )
to_dummies = ['day', 'month', 'nom_0',
'nom_1', 'nom_2', 'nom_3', 'nom_4']<categorify>
submission_df['AdoptionSpeed'].value_counts(normalize = False )
<count_missing_values><EOS> 
print(test_preds_nn.mean() , test_preds_lgb.mean() , oof_test.mean() , ffm_oof_test.mean())
 print(oof_nn.mean() , oof_lgb.mean() , oof_train.mean() , ffm_oof_train.mean() )