Kaggle competition: Google Landmark Recognition 2020
<SOS> metric: GoogleGlobalAP Kaggle data source: landmark-recognition-2020<drop_column>
plt.rcdefaults()
plt.style.use('dark_background')
 warnings.filterwarnings("ignore" )
print('FEATURE ENGINEERING FOR POS.....')
 pos=pos.groupby('SK_ID_CURR' ).agg(['mean','sum','max','var','min','median'])
 pos.columns = pd.Index([e[0] + "_" + e[1].upper() for e in pos.columns.tolist() ])
 pos.reset_index(inplace=True)
 del pos['SK_ID_PREV_MEAN']
del pos['SK_ID_PREV_MAX']
del pos['SK_ID_PREV_VAR']
del pos['SK_ID_PREV_SUM']
del pos['SK_ID_PREV_MIN']
del pos['SK_ID_PREV_MEDIAN']
print('FEATURE ENGINEERING FOR POS COMPLETED.... ' )<merge>
INPUT_PATH = os.path.join('.. ', 'input')
 DATASET_PATH = os.path.join(INPUT_PATH, 'landmark-recognition-2020')
 TRAIN_IMAGE_PATH = os.path.join(DATASET_PATH, 'train')
 TEST_IMAGE_PATH = os.path.join(DATASET_PATH, 'test')
 TRAIN_CSV_PATH = os.path.join(DATASET_PATH, 'train.csv')
 SUBMISSION_CSV_PATH = os.path.join(DATASET_PATH, 'sample_submission.csv' )
print('FEATURE ENGINEERING FOR BUR AND BAL.....')
 bal_agg = {'MONTHS_BALANCE': ['min', 'max','size','mean','var','median']}
bur_bal = bur_bal.groupby('SK_ID_BUREAU' ).agg(bal_agg)
 bur_bal.columns = pd.Index([e[0] + "_" + e[1].upper() for e in bur_bal.columns.tolist() ])
 bur_bal = bur.join(bur_bal, how='left', on='SK_ID_BUREAU')
 bur_bal.drop(columns= 'SK_ID_BUREAU', inplace= True)
 a = {
'DAYS_CREDIT': ['mean','max','sum','min','var','median'],
'CREDIT_DAY_OVERDUE': ['max','min','mean','sum','var','median'],
'DAYS_CREDIT_ENDDATE': ['mean','max','min','sum','var','median'],
'AMT_CREDIT_MAX_OVERDUE': ['mean','max','min','sum','var','median'],
'CNT_CREDIT_PROLONG': ['sum','max','min','mean','var','median'],
'AMT_CREDIT_SUM': ['mean', 'sum','max','min','var','median'],
'AMT_CREDIT_SUM_DEBT': ['mean', 'sum','max','min','var','median'],
'AMT_CREDIT_SUM_OVERDUE': ['mean','max','min','var','sum','median'],
'AMT_CREDIT_SUM_LIMIT': ['mean', 'sum','max','min','var','median'],
'DAYS_CREDIT_UPDATE': ['mean','max','min','var','sum','median'],
'AMT_ANNUITY': ['mean','max','min','sum','var','median'],
'MONTHS_BALANCE_MIN': ['min'],
'MONTHS_BALANCE_MAX': ['max'],
'MONTHS_BALANCE_SIZE': ['mean']
}
bur_bal = bur_bal.groupby('SK_ID_CURR' ).agg(a)
 bur_bal.columns = pd.Index(['BUR_' + e[0] + "_" + e[1].upper() for e in bur_bal.columns.tolist() ])
 bur_bal.reset_index(inplace=True)
 bur_bal.head()
print('FEATURE ENGINEERING FOR BUR AND BAL COMPLETED.....')
<merge>
train = pd.read_csv(TRAIN_CSV_PATH)
 print("training dataset has {} rows and {} columns".format(train.shape[0],train.shape[1]))
submission = pd.read_csv(SUBMISSION_CSV_PATH)
 print("submission dataset has {} rows and {} columns
".format(submission.shape[0],submission.shape[1]))
print('JOIN ALL DATAS ON SK_ID_CURR.... ')
 app_train = app_train.merge(right=bur_bal, how='left', on='SK_ID_CURR')
 app_test = app_test.merge(right=bur_bal, how='left', on='SK_ID_CURR')
 app_train = app_train.merge(right=prev, how='left', on='SK_ID_CURR')
 app_test = app_test.merge(right=prev, how='left', on='SK_ID_CURR')
 app_train = app_train.merge(right=pos, how='left', on='SK_ID_CURR')
 app_test = app_test.merge(right=pos, how='left', on='SK_ID_CURR')
 app_train = app_train.merge(right=credit, how='left', on='SK_ID_CURR')
 app_test = app_test.merge(right=credit, how='left', on='SK_ID_CURR')
 app_train = app_train.merge(right=install, how='left', on='SK_ID_CURR')
 app_test = app_test.merge(right=install, how='left', on='SK_ID_CURR')
 del app_train['SK_ID_CURR']
del app_test['SK_ID_CURR']
print('JOIN ALL DATAS ON SK_ID_CURR COMPLETE.... ')
<drop_column>
print(Fore.BLUE + f"{'---'*20}
Mapping for Training Data
{'---'*20}")
 data_label_dict = {'image': [], 'target': []}
for i in tqdm(range(train.shape[0])) :
data_label_dict['image'].append(
 TRAIN_IMAGE_PATH + '/' +
train['id'][i][0] + '/' +
train['id'][i][1]+ '/' +
train['id'][i][2]+ '/' +
train['id'][i] + ".jpg")
 data_label_dict['target'].append(
 train['landmark_id'][i])
 train_pathlabel = pd.DataFrame(data_label_dict)
 print(train_pathlabel.head())
 print(Fore.BLUE + f"{'---'*20}
Mapping for Test Data
{'---'*20}",Style.RESET_ALL)
 data_label_dict = {'image': []}
for i in tqdm(range(submission.shape[0])) :
data_label_dict['image'].append(
 TEST_IMAGE_PATH + '/' +
submission['id'][i][0] + '/' +
submission['id'][i][1]+ '/' +
submission['id'][i][2]+ '/' +
submission['id'][i] + ".jpg")
 test_pathlabel = pd.DataFrame(data_label_dict)
 print(test_pathlabel.head() )
train = app_train.drop(['TARGET'],axis=1)
 test = app_test
target = app_train['TARGET']
del app_train['TARGET'] 
<categorify>
print("There are", train.landmark_id.nunique() , "landmarks in the training dataset" )
print('LABEL ENCODING ALL CATEGORY COL...... ')
 var_mod = train.select_dtypes(include = 'object' ).columns.tolist()
num = train.columns.tolist()
le = LabelEncoder()
for i in var_mod:
train[i] = le.fit_transform(train[i].astype('str'))
test[i] = le.fit_transform(test[i].astype('str'))
print('LABEL ENCODING ALL CATEGORY COL COMPLETE...... ')
<prepare_x_and_y>
train.landmark_id.value_counts()
folds = KFold(n_splits=5, shuffle=True, random_state=546789)
 oof_preds = np.zeros(train.shape[0])
 sub_preds = np.zeros(test.shape[0])
 feature_importance_df = pd.DataFrame()
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(train, target)) :
train_x, train_y = train.iloc[train_idx], target.iloc[train_idx]
valid_x, valid_y = train.iloc[valid_idx], target.iloc[valid_idx]
print('PREPARING THE LGBM CLASSIFIER.....')
 clf = LGBMClassifier(
 n_estimators=10000,
learning_rate=0.03,
num_leaves=45,
colsample_bytree=0.9,
subsample=0.8,
max_depth=8,
reg_alpha=.8,
reg_lambda=.7,
min_split_gain=.01,
min_child_weight=300,
silent=-1,
verbose=-1,
class_weight='balanced',
boosting_type = 'dart',
bagging_fraction = 0.6,
bagging_freq = 100 
)
 print('CLASSIFIER IS TUNNED.... ')
 clf.fit(train_x, train_y,
eval_set= [(train_x, train_y),(valid_x, valid_y)],
eval_metric='auc', verbose=100, early_stopping_rounds=100, feature_name = num, categorical_feature=var_mod 
)
 oof_preds[valid_idx] = clf.predict_proba(valid_x, num_iteration=clf.best_iteration_)[:, 1]
sub_preds += clf.predict_proba(test, num_iteration=clf.best_iteration_)[:, 1] / folds.n_splits
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = train_x.columns.tolist()
fold_importance_df["importance"] = clf.feature_importances_
fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 print('Fold %2d AUC : %.6f' %(n_fold + 1, roc_auc_score(valid_y, oof_preds[valid_idx])))
 del train_x, train_y, valid_x, valid_y
gc.collect()<train_model>
import copy
import csv
import gc
import operator
import os
import pathlib
import shutil
import numpy as np
import PIL
import pydegensac
from scipy import spatial
import tensorflow as tf
print('FIT MODEL WITH TOP 100 FEATURES.....')
 cols = best_features.feature.tolist()
train = train[cols]
test = test[cols]
clf.fit(train,target )<predict_on_test>
INPUT_DIR = os.path.join('.. ', 'input')
 DATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')
 TEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')
 TRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')
 TRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv' )
print('PREDICTING TEST DATA...... ')
 pred = clf.predict_proba(test,num_iteration=clf.best_iteration_)[:,1]<prepare_output>
NUM_PUBLIC_TRAIN_IMAGES = 1580470
MAX_NUM_EMBEDDINGS = -1
sub= app_test1
sub['TARGET'] = p5
t2 = sub[['SK_ID_CURR','TARGET']]
t2.head()<count_missing_values>
NUM_TO_RERANK = 3
TOP_K = 3
print(t2.isnull().sum())
 t2.head()<save_to_csv>
MAX_INLIER_SCORE = 35
MAX_REPROJECTION_ERROR = 7.0
MAX_RANSAC_ITERATIONS = 8500000
HOMOGRAPHY_CONFIDENCE = 0.99
t2.to_csv('submission.csv',index=False )<set_options>
SAVED_MODEL_DIR = '.. /input/delg-saved-models/local_and_global'
DELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)
 DELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])
 DELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)
DELG_INPUT_TENSOR_NAMES = [
'input_image:0', 'input_scales:0', 'input_abs_thres:0'
]
warnings.filterwarnings("ignore")
 warnings.simplefilter(action='ignore', category=FutureWarning)
<import_modules>
NUM_EMBEDDING_DIMENSIONS = 2048
GLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,
['global_descriptors:0'])
 LOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)
 LOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(
 DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],
['boxes:0', 'features:0'] )
from keras.models import *
from keras.optimizers import *
from keras.regularizers import *
from keras.layers import *
from keras import backend as K
from keras.callbacks import *<load_from_csv>
def to_hex(image_id)-> str:
return '{0:0{1}x}'.format(image_id, 16)
 def get_image_path(subset, image_id):
name = to_hex(image_id)
 return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],
'{}.jpg'.format(name))
def load_image_tensor(image_path):
return tf.convert_to_tensor(
 np.array(PIL.Image.open(image_path ).convert('RGB')))
 def extract_global_features(image_root_dir):

 image_paths = [x for x in pathlib.Path(image_root_dir ).rglob('*.jpg')]
num_embeddings = len(image_paths)
 if MAX_NUM_EMBEDDINGS > 0:
num_embeddings = min(MAX_NUM_EMBEDDINGS, num_embeddings)
 ids = num_embeddings * [None]
embeddings = np.empty(( num_embeddings, NUM_EMBEDDING_DIMENSIONS))
for i, image_path in enumerate(image_paths):
if i >= num_embeddings:
break
ids[i] = int(image_path.name.split('.')[0], 16)
 image_tensor = load_image_tensor(image_path)
 features = GLOBAL_FEATURE_EXTRACTION_FN(image_tensor,
DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR)
 embeddings[i, :] = tf.nn.l2_normalize(
 tf.reduce_sum(features[0], axis=0, name='sum_pooling'),
axis=0,
name='final_l2_normalization' ).numpy()
return ids, embeddings
def extract_local_features(image_path):

 image_tensor = load_image_tensor(image_path)
 features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR,
LOCAL_FEATURE_NUM_TENSOR)
 keypoints = tf.divide(
 tf.add(
 tf.gather(features[0], [0, 1], axis=1),
tf.gather(features[0], [2, 3], axis=1)) , 2.0 ).numpy()
descriptors = tf.nn.l2_normalize(
 features[1], axis=1, name='l2_normalization' ).numpy()
return keypoints, descriptors
def get_putative_matching_keypoints(test_keypoints,
test_descriptors,
train_keypoints,
train_descriptors,
max_distance=0.9):

 train_descriptor_tree = spatial.cKDTree(train_descriptors)
 _, matches = train_descriptor_tree.query(
 test_descriptors, distance_upper_bound=max_distance)
 test_kp_count = test_keypoints.shape[0]
train_kp_count = train_keypoints.shape[0]
test_matching_keypoints = np.array([
test_keypoints[i,]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 train_matching_keypoints = np.array([
train_keypoints[matches[i],]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 return test_matching_keypoints, train_matching_keypoints
def get_num_inliers(test_keypoints, test_descriptors, train_keypoints,
train_descriptors):

 test_match_kp, train_match_kp = get_putative_matching_keypoints(
 test_keypoints, test_descriptors, train_keypoints, train_descriptors)
 if test_match_kp.shape[
0] <= 4:
return 0
try:
_, mask = pydegensac.findHomography(test_match_kp, train_match_kp,
MAX_REPROJECTION_ERROR,
HOMOGRAPHY_CONFIDENCE,
MAX_RANSAC_ITERATIONS)
 except np.linalg.LinAlgError:
return 0
return int(copy.deepcopy(mask ).astype(np.float32 ).sum())
 def get_total_score(num_inliers, global_score):
local_score = min(num_inliers, MAX_INLIER_SCORE)/ MAX_INLIER_SCORE
return local_score + global_score
def rescore_and_rerank_by_num_inliers(test_image_id,
train_ids_labels_and_scores):

 test_image_path = get_image_path('test', test_image_id)
 test_keypoints, test_descriptors = extract_local_features(test_image_path)
 for i in range(len(train_ids_labels_and_scores)) :
train_image_id, label, global_score = train_ids_labels_and_scores[i]
train_image_path = get_image_path('train', train_image_id)
 train_keypoints, train_descriptors = extract_local_features(
 train_image_path)
 num_inliers = get_num_inliers(test_keypoints, test_descriptors,
train_keypoints, train_descriptors)
 total_score = get_total_score(num_inliers, global_score)
 train_ids_labels_and_scores[i] =(train_image_id, label, total_score)
 train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)
 return train_ids_labels_and_scores
def load_labelmap() :
with open(TRAIN_LABELMAP_PATH, mode='r')as csv_file:
csv_reader = csv.DictReader(csv_file)
 labelmap = {row['id']: row['landmark_id'] for row in csv_reader}
return labelmap
def get_prediction_map(test_ids, train_ids_labels_and_scores):

 prediction_map = dict()
for test_index, test_id in enumerate(test_ids):
hex_test_id = to_hex(test_id)
 aggregate_scores = {}
for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:
if label not in aggregate_scores:
aggregate_scores[label] = 0
aggregate_scores[label] += score
label, score = max(aggregate_scores.items() , key=operator.itemgetter(1))
prediction_map[hex_test_id] = {'score': score, 'class': label}
return prediction_map
def get_predictions(labelmap):

 test_ids, test_embeddings = extract_global_features(TEST_IMAGE_DIR)
 train_ids, train_embeddings = extract_global_features(TRAIN_IMAGE_DIR)
 train_ids_labels_and_scores = [None] * test_embeddings.shape[0]
for test_index in range(test_embeddings.shape[0]):
distances = spatial.distance.cdist(
 test_embeddings[np.newaxis, test_index, :], train_embeddings,
'cosine')[0]
partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]
nearest = sorted([(train_ids[p], distances[p])for p in partition],
key=lambda x: x[1])
 train_ids_labels_and_scores[test_index] = [ 
(train_id, labelmap[to_hex(train_id)], 1.- cosine_distance)
 for train_id, cosine_distance in nearest
]
del test_embeddings
del train_embeddings
del labelmap
gc.collect()
pre_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 for test_index, test_id in enumerate(test_ids):
train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(
 test_id, train_ids_labels_and_scores[test_index])
 post_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 return pre_verification_predictions, post_verification_predictions
def save_submission_csv(predictions=None):

 if predictions is None:
shutil.copyfile(
 os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')
 return
with open('submission.csv', 'w')as submission_csv:
csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])
 csv_writer.writeheader()
for image_id, prediction in predictions.items() :
label = prediction['class']
score = prediction['score']
csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'} )
print(os.listdir('.. /input/'))
train_X_0 = pd.read_csv(".. /input/lanl-master-s-features-creating-0/train_X_features_865.csv")
 train_X_1 = pd.read_csv(".. /input/lanl-master-s-features-creating-1/train_X_features_865.csv")
 y_0 = pd.read_csv(".. /input/lanl-master-s-features-creating-0/train_y.csv", index_col=False, header=None)
 y_1 = pd.read_csv(".. /input/lanl-master-s-features-creating-1/train_y.csv", index_col=False, header=None )<concatenate>
def main() :
labelmap = load_labelmap()
num_training_images = len(labelmap.keys())
 print(f'Found {num_training_images} training images.')
 if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:
print(
 f'Found {NUM_PUBLIC_TRAIN_IMAGES} training images.Copying sample submission.' 
)
 save_submission_csv()
return
_, post_verification_predictions = get_predictions(labelmap)
 save_submission_csv(post_verification_predictions)
 main()