Kaggle competition: Google Landmark Recognition 2020
<SOS> metric: GoogleGlobalAP Kaggle data source: landmark-recognition-2020<compute_train_metric>
!pip install.. /input/glrec2020/Keras_Applications-1.0.8-py3-none-any.whl
!pip install.. /input/glrec2020/efficientnet-1.1.0-py3-none-any.whl
second_round_zero_features, feature_importances = identify_zero_importance_features(train, train_labels )<filter>
import operator
import gc
import pathlib
import shutil
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras import backend as K
from scipy import spatial
import cv2
import efficientnet.tfkeras as efn
import math
import copy
import csv
import os
import numpy as np
import PIL
import pydegensac
threshold = 0.95
features_to_keep = list(norm_feature_importances[norm_feature_importances['cumulative_importance'] < threshold]['feature'])
 train_small = train[features_to_keep]
test_small = test[features_to_keep]<save_to_csv>
NUMBER_OF_CLASSES = 81313
IMAGE_SIZE = [384, 384]
LR = 0.0001
train_small['TARGET'] = train_labels
train_small['SK_ID_CURR'] = train_ids
test_small['SK_ID_CURR'] = test_ids
train_small.to_csv('m_train_small.csv', index = False)
 test_small.to_csv('m_test_small.csv', index = False )<train_model>
MODEL1 = get_model_B6()
MODEL2 = get_model_B7()
MODEL1.load_weights('.. /input/effb6-512-ep18/effb6model512-18.h5')
 MODEL1 = tf.keras.models.Model(inputs = MODEL1.input[0], outputs = MODEL1.layers[-4].output)
 MODEL2.load_weights('.. /input/effb7-512-ep12/effb7model512-12.h5')
 MODEL2 = tf.keras.models.Model(inputs = MODEL2.input[0], outputs = MODEL2.layers[-4].output )
def model(features, test_features, encoding = 'ohe', n_folds = 5):

 train_ids = features['SK_ID_CURR']
test_ids = test_features['SK_ID_CURR']
labels = features['TARGET']
features = features.drop(columns = ['SK_ID_CURR', 'TARGET'])
 test_features = test_features.drop(columns = ['SK_ID_CURR'])
 if encoding == 'ohe':
features = pd.get_dummies(features)
 test_features = pd.get_dummies(test_features)
 features, test_features = features.align(test_features, join = 'inner', axis = 1)
 cat_indices = 'auto'
elif encoding == 'le':
label_encoder = LabelEncoder()
cat_indices = []
for i, col in enumerate(features):
if features[col].dtype == 'object':
features[col] = label_encoder.fit_transform(np.array(features[col].astype(str)).reshape(( -1,)))
 test_features[col] = label_encoder.transform(np.array(test_features[col].astype(str)).reshape(( -1,)))
 cat_indices.append(i)
 else:
raise ValueError("Encoding must be either 'ohe' or 'le'")
 print('Training Data Shape: ', features.shape)
 print('Testing Data Shape: ', test_features.shape)
 feature_names = list(features.columns)
 features = np.array(features)
 test_features = np.array(test_features)
 k_fold = KFold(n_splits = n_folds, shuffle = False, random_state = 50)
 feature_importance_values = np.zeros(len(feature_names))
test_predictions = np.zeros(test_features.shape[0])
 out_of_fold = np.zeros(features.shape[0])
 valid_scores = []
train_scores = []
for train_indices, valid_indices in k_fold.split(features):
train_features, train_labels = features[train_indices], labels[train_indices]
valid_features, valid_labels = features[valid_indices], labels[valid_indices]
model = lgb.LGBMClassifier(n_estimators=10000, objective = 'binary', boosting_type='goss',
class_weight = 'balanced', learning_rate = 0.05,
reg_alpha = 0.1, reg_lambda = 0.1, n_jobs = -1, random_state = 50)
 model.fit(train_features, train_labels, eval_metric = 'auc',
eval_set = [(valid_features, valid_labels),(train_features, train_labels)],
eval_names = ['valid', 'train'], categorical_feature = cat_indices,
early_stopping_rounds = 100, verbose = 200)
 best_iteration = model.best_iteration_
feature_importance_values += model.feature_importances_ / k_fold.n_splits
test_predictions += model.predict_proba(test_features, num_iteration = best_iteration)[:, 1] / k_fold.n_splits
out_of_fold[valid_indices] = model.predict_proba(valid_features, num_iteration = best_iteration)[:, 1]
valid_score = model.best_score_['valid']['auc']
train_score = model.best_score_['train']['auc']
valid_scores.append(valid_score)
 train_scores.append(train_score)
 gc.enable()
del model, train_features, valid_features
gc.collect()
submission = pd.DataFrame({'SK_ID_CURR': test_ids, 'TARGET': test_predictions})
 feature_importances = pd.DataFrame({'feature': feature_names, 'importance': feature_importance_values})
 valid_auc = roc_auc_score(labels, out_of_fold)
 valid_scores.append(valid_auc)
 train_scores.append(np.mean(train_scores))
fold_names = list(range(n_folds))
fold_names.append('overall')
 metrics = pd.DataFrame({'fold': fold_names,
'train': train_scores,
'valid': valid_scores})
 return submission, feature_importances, metrics<prepare_output>
NUM_TO_RERANK = 3
NUM_PUBLIC_TEST_IMAGES = 10345
train['TARGET'] = train_labels
train['SK_ID_CURR'] = train_ids
test['SK_ID_CURR'] = test_ids
submission, feature_importances, metrics = model(train, test )<save_to_csv>
INPUT_DIR = os.path.join('.. ', 'input')
 DATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')
 TEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')
 TRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')
 TRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv' )
submission.to_csv('selected_features_submission.csv', index = False )<compute_train_metric>
NUM_PUBLIC_TRAIN_IMAGES = 1580470
MAX_NUM_EMBEDDINGS = -1
NUM_TO_RERANK = 3
TOP_K = 3
MAX_INLIER_SCORE = 25
MAX_REPROJECTION_ERROR = 7.0
MAX_RANSAC_ITERATIONS = 100000
HOMOGRAPHY_CONFIDENCE = 0.99
submission_small, feature_importances_small, metrics_small = model(train_small, test_small )<save_to_csv>
SAVED_MODEL_DIR = '.. /input/delg-saved-models/local_and_global'
DELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)
 DELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 2])
 DELG_SCORE_THRESHOLD_TENSOR = tf.constant(375.)
DELG_INPUT_TENSOR_NAMES = [
'input_image:0', 'input_scales:0', 'input_abs_thres:0'
]
NUM_EMBEDDING_DIMENSIONS = 3072
GLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,
['global_descriptors:0'])
 LOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)
 LOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(
 DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],
['boxes:0', 'features:0'] )
<categorify><EOS> 
def to_hex(image_id)-> str:
return '{0:0{1}x}'.format(image_id, 16)
 def get_image_path(subset, image_id):
name = to_hex(image_id)
 return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],
'{}.jpg'.format(name))
def load_image_tensor(image_path):
return tf.convert_to_tensor(
 np.array(PIL.Image.open(image_path ).convert('RGB')) )