Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<train_model>
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
pd.set_option('display.max_columns', 99)
 pd.set_option('display.max_rows', 99)
folds = StratifiedKFold(n_splits=N_FOLDS, shuffle=True, random_state=42)
 oof = np.zeros(len(train_selected))
sub = np.zeros(len(test_selected))
score = [0 for _ in range(folds.n_splits)]
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_selected.values, target)) :
X_train, y_train = train_selected.iloc[trn_idx][features], target[trn_idx]
X_val, y_val = train_selected.iloc[val_idx][features], target[val_idx]
X_test = test_selected.values
clf = LogisticRegression(penalty="l1", C=0.1, solver="liblinear", random_state=42)
 clf.fit(X_train, y_train)
 oof[val_idx] = clf.predict_proba(X_val)[:, 1]
sub += clf.predict_proba(X_test)[:, 1] / folds.n_splits
score[fold_] = roc_auc_score(target[val_idx], oof[val_idx])
 print("Fold {}: {}".format(fold_+1, round(score[fold_],5)))
 print("CV score(auc): {:<8.5f},(std: {:<8.5f})".format(roc_auc_score(target, oof), np.std(score)) )<save_to_csv>
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
sns.set_palette(sns.color_palette('tab20', 20))
submit = pd.read_csv('.. /input/sample_submission.csv')
 submit["target"] = sub
submit.to_csv("submission.csv", index=False)
 submit.head(20 )<import_modules>
COMP = '.. /input/covid19-global-forecasting-week-2'
DATEFORMAT = '%Y-%m-%d'
def get_comp_data(COMP):
train = pd.read_csv(f'{COMP}/train.csv')
 test = pd.read_csv(f'{COMP}/test.csv')
 submission = pd.read_csv(f'{COMP}/submission.csv')
 print(train.shape, test.shape, submission.shape)
 train['Country_Region'] = train['Country_Region'].str.replace(',', '')
 test['Country_Region'] = test['Country_Region'].str.replace(',', '')
 train['Location'] = train['Country_Region'] + '-' + train['Province_State'].fillna('')
 test['Location'] = test['Country_Region'] + '-' + test['Province_State'].fillna('')
 train['LogConfirmed'] = to_log(train.ConfirmedCases)
 train['LogFatalities'] = to_log(train.Fatalities)
 train = train.drop(columns=['Province_State'])
 test = test.drop(columns=['Province_State'])
 country_codes = pd.read_csv('.. /input/covid19-metadata/country_codes.csv', keep_default_na=False)
 train = train.merge(country_codes, on='Country_Region', how='left')
 test = test.merge(country_codes, on='Country_Region', how='left')
 train['DateTime'] = pd.to_datetime(train['Date'])
 test['DateTime'] = pd.to_datetime(test['Date'])
 return train, test, submission
def process_each_location(df):
dfs = []
for loc, df in tqdm(df.groupby('Location')) :
df = df.sort_values(by='Date')
 df['Fatalities'] = df['Fatalities'].cummax()
df['ConfirmedCases'] = df['ConfirmedCases'].cummax()
df['LogFatalities'] = df['LogFatalities'].cummax()
df['LogConfirmed'] = df['LogConfirmed'].cummax()
df['LogConfirmedNextDay'] = df['LogConfirmed'].shift(-1)
 df['ConfirmedNextDay'] = df['ConfirmedCases'].shift(-1)
 df['DateNextDay'] = df['Date'].shift(-1)
 df['LogFatalitiesNextDay'] = df['LogFatalities'].shift(-1)
 df['FatalitiesNextDay'] = df['Fatalities'].shift(-1)
 df['LogConfirmedDelta'] = df['LogConfirmedNextDay'] - df['LogConfirmed']
df['ConfirmedDelta'] = df['ConfirmedNextDay'] - df['ConfirmedCases']
df['LogFatalitiesDelta'] = df['LogFatalitiesNextDay'] - df['LogFatalities']
df['FatalitiesDelta'] = df['FatalitiesNextDay'] - df['Fatalities']
dfs.append(df)
 return pd.concat(dfs)
 def add_days(d, k):
return dt.datetime.strptime(d, DATEFORMAT)+ dt.timedelta(days=k)
 def to_log(x):
return np.log(x + 1)
 def to_exp(x):
return np.exp(x)- 1
gc.enable()
tf.compat.v1.enable_resource_variables()
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
 print(f"Tensorflow Version: {tf.__version__}" )<define_variables>
train = train.sort_values(by='Date')
 countries_latest_state = train[train['Date'] == TRAIN_END].groupby([
'Country_Region', 'continent', 'geo_region', 'country_iso_code_3'] ).sum() [[
'ConfirmedCases', 'Fatalities']].reset_index()
countries_latest_state['Log10Confirmed'] = np.log10(countries_latest_state.ConfirmedCases + 1)
 countries_latest_state['Log10Fatalities'] = np.log10(countries_latest_state.Fatalities + 1)
 countries_latest_state = countries_latest_state.sort_values(by='Fatalities', ascending=False)
 countries_latest_state.to_csv('countries_latest_state.csv', index=False)
 countries_latest_state.shape
countries_latest_state.head()
def set_seed(seed: int = 42):
random.seed(seed)
 np.random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 tf.random.set_seed(seed)
 rand_seed = 1120
set_seed(rand_seed)
 debug_mode = False
NUM_PUBLIC_TRAIN_IMAGES = 1580470
if debug_mode:
MAX_NUM_EMBEDDINGS = 10
NUM_PUBLIC_TRAIN_IMAGES = 10
else:
MAX_NUM_EMBEDDINGS = -1<define_variables>
latest_loc = train[train['Date'] == TRAIN_END][['Location', 'ConfirmedCases', 'Fatalities']]
max_loc = train.groupby(['Location'])[['ConfirmedCases', 'Fatalities']].max().reset_index()
check = pd.merge(latest_loc, max_loc, on='Location')
 np.mean(check.ConfirmedCases_x == check.ConfirmedCases_y)
 np.mean(check.Fatalities_x == check.Fatalities_y)
 check[check.Fatalities_x != check.Fatalities_y]
check[check.ConfirmedCases_x != check.ConfirmedCases_y]
INPUT_DIR = os.path.join('.. ', 'input')
 DATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')
 TEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')
 TRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')
 TRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')
 NUM_TO_RERANK = 10
TOP_K = 10
MAX_INLIER_SCORE = 70
MAX_REPROJECTION_ERROR = 4.0
MAX_RANSAC_ITERATIONS = 10000
HOMOGRAPHY_CONFIDENCE = 0.995 
<define_variables>
regional_progress = train_clean.groupby(['DateTime', 'continent'] ).sum() [['ConfirmedCases', 'Fatalities']].reset_index()
regional_progress['Log10Confirmed'] = np.log10(regional_progress.ConfirmedCases + 1)
 regional_progress['Log10Fatalities'] = np.log10(regional_progress.Fatalities + 1)
 regional_progress = regional_progress[regional_progress.continent != '
models = ["r50", "r101", "rn101af"]
model_config = {
"r50": {
"SAVED_MODEL_DIR":
".. /input/pretrained-delg/r50delg_gldv2clean_20200914",
"DELG_SCORE_THRESHOLD_TENSOR": tf.constant(454.6),
"global_weight": 0.4,
"global_only": False,
"is_tf2": False,
},
"r101": {
"SAVED_MODEL_DIR":
".. /input/pretrained-delg/r101delg_gldv2clean_20200914",
"DELG_SCORE_THRESHOLD_TENSOR": tf.constant(357.48),
"global_weight": 0.4,
"global_only": False,
"is_tf2": False,
},
"rn101af": {
"SAVED_MODEL_DIR":
".. /input/pretrained-delg/rn101_af_gldv2clean_20200814",
"global_weight": 0.2,
"global_only": True,
"is_tf2": True,
}
}
DELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0,
1.4142135])
 LOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)
 NUM_EMBEDDING_DIMENSIONS = 2048
GLOBAL_SCALES_IND_TENSOR = tf.range(
 len(DELG_IMAGE_SCALES_TENSOR))<categorify>
countries_0301 = country_progress[country_progress.Date == '2020-03-01'][[
'Country_Region', 'ConfirmedCases', 'Fatalities']]
countries_0331 = country_progress[country_progress.Date == '2020-03-31'][[
'Country_Region', 'ConfirmedCases', 'Fatalities']]
countries_in_march = pd.merge(countries_0301, countries_0331, on='Country_Region', suffixes=['_0301', '_0331'])
 countries_in_march['IncreaseInMarch'] = countries_in_march.ConfirmedCases_0331 /(countries_in_march.ConfirmedCases_0301 + 1)
 countries_in_march = countries_in_march[countries_in_march.ConfirmedCases_0331 > 200].sort_values(
 by='IncreaseInMarch', ascending=False)
 countries_in_march.tail(15 )
def to_hex(image_id)-> str:
return '{0:0{1}x}'.format(image_id, 16)
 def get_image_path(subset, image_id):
name = to_hex(image_id)
 return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],
'{}.jpg'.format(name))
def load_image_tensor(image_path):
return tf.convert_to_tensor(
 np.array(PIL.Image.open(image_path ).convert('RGB')))
 def extract_global_features(image_root_dir, extractor, is_tf2=False):

 image_paths = [x for x in pathlib.Path(image_root_dir ).rglob('*.jpg')]
num_embeddings = len(image_paths)
 if MAX_NUM_EMBEDDINGS > 0:
num_embeddings = min(MAX_NUM_EMBEDDINGS, num_embeddings)
 print(
 f"Extracting global features from {num_embeddings} images(dim={NUM_EMBEDDING_DIMENSIONS} )...... " 
)
 ids = num_embeddings * [None]
embeddings = np.empty(( num_embeddings, NUM_EMBEDDING_DIMENSIONS))
for i, image_path in enumerate(image_paths):
if i >= num_embeddings:
break
ids[i] = int(image_path.name.split('.')[0], 16)
 image_tensor = load_image_tensor(image_path)
 if not is_tf2:
output = extractor(
 input_image=image_tensor,
input_scales=DELG_IMAGE_SCALES_TENSOR,
input_global_scales_ind=GLOBAL_SCALES_IND_TENSOR)
 raw_global_descriptors = output[-1]
else:
output_dict = extractor(
 input_image=image_tensor,
input_scales=DELG_IMAGE_SCALES_TENSOR,
input_global_scales_ind=GLOBAL_SCALES_IND_TENSOR)
 raw_global_descriptors = output_dict["global_descriptors"]
embedding_tensor = tf.nn.l2_normalize(raw_global_descriptors,
axis=1,
name='l2_normalization')
 embedding_tensor = tf.reduce_sum(embedding_tensor,
axis=0,
name='sum_pooling')
 embeddings[i, :] = tf.nn.l2_normalize(
 embedding_tensor, axis=0, name='final_l2_normalization' ).numpy()
return ids, embeddings
def extract_local_features(model_name, image_path, extractor):

 image_tensor = load_image_tensor(image_path)
 output = extractor(input_image=image_tensor,
input_scales=DELG_IMAGE_SCALES_TENSOR,
input_global_scales_ind=GLOBAL_SCALES_IND_TENSOR,
input_max_feature_num=LOCAL_FEATURE_NUM_TENSOR,
input_abs_thres=model_config[model_name]
["DELG_SCORE_THRESHOLD_TENSOR"])
 boxes, features = output[0], output[1]
keypoints = tf.divide(
 tf.add(tf.gather(boxes, [0, 1], axis=1),
tf.gather(boxes, [2, 3], axis=1)) , 2.0 ).numpy()
local_descriptors = tf.nn.l2_normalize(features,
axis=1,
name='l2_normalization' ).numpy()
return keypoints, local_descriptors<compute_train_metric>
train_clean['Geo
latest = train_clean[train_clean.Date == '2020-03-31'][[
'Geo
daily_confirmed_deltas = train_clean[train_clean.Date >= '2020-03-17'].pivot(
 'Geo
daily_confirmed_deltas = latest.merge(daily_confirmed_deltas, on='Geo
daily_confirmed_deltas.shape
daily_confirmed_deltas.head()
daily_confirmed_deltas.to_csv('daily_confirmed_deltas.csv', index=False )
def compute_putative_matching_keypoints(test_keypoints,
test_descriptors,
train_keypoints,
train_descriptors,
max_distance=0.9):

 train_descriptor_tree = spatial.cKDTree(train_descriptors)
 _, matches = train_descriptor_tree.query(test_descriptors,
distance_upper_bound=max_distance)
 test_kp_count = test_keypoints.shape[0]
train_kp_count = train_keypoints.shape[0]
test_matching_keypoints = np.array([
test_keypoints[i, ] for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 train_matching_keypoints = np.array([
train_keypoints[matches[i], ] for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 return test_matching_keypoints, train_matching_keypoints
def compute_num_inliers(test_keypoints, test_descriptors, train_keypoints,
train_descriptors):

 test_match_kp, train_match_kp = compute_putative_matching_keypoints(
 test_keypoints, test_descriptors, train_keypoints, train_descriptors)
 if test_match_kp.shape[
0] <= 4:
return 0
try:
_, mask = pydegensac.findHomography(test_match_kp, train_match_kp,
MAX_REPROJECTION_ERROR,
HOMOGRAPHY_CONFIDENCE,
MAX_RANSAC_ITERATIONS)
 except np.linalg.LinAlgError:
return 0
return int(copy.deepcopy(mask ).astype(np.float32 ).sum())
 def get_total_score(num_inliers, global_score):
local_score = min(num_inliers, MAX_INLIER_SCORE)/ MAX_INLIER_SCORE
return local_score + global_score
def rescore_and_rerank_by_num_inliers(test_image_id,
train_ids_labels_and_scores,
local_models):

 test_image_path = get_image_path('test', test_image_id)
 model_name = "r50"
extractor = local_models[model_name]
test_keypoints, test_descriptors = extract_local_features(
 model_name, test_image_path, extractor)
 for i in range(len(train_ids_labels_and_scores)) :
train_image_id, label, global_score = train_ids_labels_and_scores[i]
train_image_path = get_image_path('train', train_image_id)
 train_keypoints, train_descriptors = extract_local_features(
 model_name, train_image_path, extractor)
 num_inliers = compute_num_inliers(test_keypoints, test_descriptors,
train_keypoints, train_descriptors)
 total_score = get_total_score(num_inliers, global_score)
 train_ids_labels_and_scores[i] =(train_image_id, label, total_score)
 train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)
 return train_ids_labels_and_scores 
<choose_model_class>
deltas = train_clean[np.logical_and(
 train_clean.LogConfirmed > 2,
~train_clean.Location.str.startswith('China')
)].dropna().sort_values(by='LogConfirmedDelta', ascending=False)
 deltas['start'] = deltas['LogConfirmed'].round(0)
 confirmed_deltas = pd.concat([
deltas.groupby('start')[['LogConfirmedDelta']].mean() ,
deltas.groupby('start')[['LogConfirmedDelta']].std() ,
deltas.groupby('start')[['LogConfirmedDelta']].count()
], axis=1)
 deltas.mean()
confirmed_deltas.columns = ['avg', 'std', 'cnt']
confirmed_deltas
confirmed_deltas.to_csv('confirmed_deltas.csv' )
def get_global_extractor(model, is_tf2=False):
if not is_tf2:
DELG_INPUT_TENSOR_NAMES = [
'input_image:0', 'input_scales:0', 'input_global_scales_ind:0'
]
GLOBAL_FEATURE_EXTRACTION_FN = model.prune(DELG_INPUT_TENSOR_NAMES,
['global_descriptors:0'])
 return GLOBAL_FEATURE_EXTRACTION_FN
else:
return model.signatures['serving_default']
def get_local_extractor(model):
DELG_INPUT_TENSOR_NAMES = [
'input_image:0', 'input_scales:0', 'input_global_scales_ind:0'
]
LOCAL_FEATURE_EXTRACTION_FN = model.prune(
 DELG_INPUT_TENSOR_NAMES +
['input_max_feature_num:0', 'input_abs_thres:0'],
['boxes:0', 'features:0'])
 return LOCAL_FEATURE_EXTRACTION_FN<load_from_csv>
DECAY = 0.93
DECAY ** 7, DECAY ** 14, DECAY ** 21, DECAY ** 28
confirmed_deltas = train.groupby(['Location', 'Country_Region', 'continent'])[[
'Id']].count().reset_index()
GLOBAL_DELTA = 0.11
confirmed_deltas['DELTA'] = GLOBAL_DELTA
confirmed_deltas.loc[confirmed_deltas.continent=='Africa', 'DELTA'] = 0.14
confirmed_deltas.loc[confirmed_deltas.continent=='Oceania', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Korea South', 'DELTA'] = 0.011
confirmed_deltas.loc[confirmed_deltas.Country_Region=='US', 'DELTA'] = 0.15
confirmed_deltas.loc[confirmed_deltas.Country_Region=='China', 'DELTA'] = 0.01
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Japan', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Singapore', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Taiwan*', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Switzerland', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Norway', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Iceland', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Austria', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Italy', 'DELTA'] = 0.04
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Spain', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Portugal', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Israel', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Iran', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Germany', 'DELTA'] = 0.07
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Malaysia', 'DELTA'] = 0.06
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Russia', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Ukraine', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Brazil', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Turkey', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Country_Region=='Philippines', 'DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.Location=='France-', 'DELTA'] = 0.1
confirmed_deltas.loc[confirmed_deltas.Location=='United Kingdom-', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.Location=='Diamond Princess-', 'DELTA'] = 0.00
confirmed_deltas.loc[confirmed_deltas.Location=='China-Hong Kong', 'DELTA'] = 0.08
confirmed_deltas.loc[confirmed_deltas.Location=='San Marino-', 'DELTA'] = 0.03
confirmed_deltas.shape, confirmed_deltas.DELTA.mean()
confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA].shape, confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA].DELTA.mean()
confirmed_deltas[confirmed_deltas.DELTA != GLOBAL_DELTA]
confirmed_deltas.describe()
def load_labelmap() :
with open(TRAIN_LABELMAP_PATH, mode='r')as csv_file:
csv_reader = csv.DictReader(csv_file)
 labelmap = {row['id']: row['landmark_id'] for row in csv_reader}
return labelmap
def get_prediction_map(test_ids, train_ids_labels_and_scores):

 prediction_map = dict()
for test_index, test_id in enumerate(test_ids):
hex_test_id = to_hex(test_id)
 aggregate_scores = {}
for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:
if label not in aggregate_scores:
aggregate_scores[label] = 0
aggregate_scores[label] += score
label, score = max(aggregate_scores.items() ,
key=operator.itemgetter(1))
prediction_map[hex_test_id] = {'score': score, 'class': label}
return prediction_map
def get_predictions(labelmap, global_models, local_models):

 test_ids, train_ids = None, None
model_test_embeds, model_train_embeds = [], []
for model_name in models:
global_extractor = global_models[model_name]
test_ids, model_test_embeddings = extract_global_features(
 TEST_IMAGE_DIR, global_extractor,
model_config[model_name]["is_tf2"])
 train_ids, model_train_embeddings = extract_global_features(
 TRAIN_IMAGE_DIR, global_extractor,
model_config[model_name]["is_tf2"])
 model_test_embeds.append(model_test_embeddings)
 model_train_embeds.append(model_train_embeddings)
 del global_extractor
gc.collect()
tf.keras.backend.clear_session()
test_embeddings = np.zeros_like(model_test_embeds[0])
 train_embeddings = np.zeros_like(model_train_embeds[0])
 for i in range(len(models)) :
test_embeddings += model_config[model_name][
"global_weight"] * model_test_embeds[i]
train_embeddings += model_config[model_name][
"global_weight"] * model_train_embeds[i]
train_ids_labels_and_scores = [None] * test_embeddings.shape[0]
for test_index in range(test_embeddings.shape[0]):
distances = spatial.distance.cdist(
 test_embeddings[np.newaxis, test_index, :], train_embeddings,
'cosine')[0]
partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]
nearest = sorted([(train_ids[p], distances[p])for p in partition],
key=lambda x: x[1])
 train_ids_labels_and_scores[test_index] = [ 
(train_id, labelmap[to_hex(train_id)], 1.- cosine_distance)
 for train_id, cosine_distance in nearest
]
del test_embeddings
del train_embeddings
del labelmap
gc.collect()
pre_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 for test_index, test_id in enumerate(test_ids):
train_ids_labels_and_scores[
test_index] = rescore_and_rerank_by_num_inliers(
 test_id, train_ids_labels_and_scores[test_index], local_models)
 post_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 return pre_verification_predictions, post_verification_predictions<save_to_csv>
daily_log_confirmed = train_clean.pivot('Location', 'Date', 'LogConfirmed' ).reset_index()
daily_log_confirmed = daily_log_confirmed.sort_values(TRAIN_END, ascending=False)
 daily_log_confirmed.to_csv('daily_log_confirmed.csv', index=False)
 for i, d in tqdm(enumerate(pd.date_range(add_days(TRAIN_END, 1), add_days(TEST_END, 1)))) :
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in confirmed_deltas.Location.values:
confirmed_delta = confirmed_deltas.loc[confirmed_deltas.Location == loc, 'DELTA'].values[0]
daily_log_confirmed.loc[daily_log_confirmed.Location == loc, new_day] = daily_log_confirmed.loc[daily_log_confirmed.Location == loc, last_day] + \
confirmed_delta * DECAY ** i
def save_submission_csv(predictions=None):

 if predictions is None:
shutil.copyfile(os.path.join(DATASET_DIR, 'sample_submission.csv'),
'submission.csv')
 return
with open('submission.csv', 'w')as submission_csv:
csv_writer = csv.DictWriter(submission_csv,
fieldnames=['id', 'landmarks'])
 csv_writer.writeheader()
for image_id, prediction in predictions.items() :
label = prediction['class']
score = prediction['score']
csv_writer.writerow({
'id': image_id,
'landmarks': f'{label} {score}'
} )<load_pretrained>
train_clean['Geo
latest = train_clean[train_clean.Date == TRAIN_END][[
'Geo
daily_death_deltas = train_clean[train_clean.Date >= '2020-03-17'].pivot(
 'Geo
daily_death_deltas = latest.merge(daily_death_deltas, on='Geo
daily_death_deltas.shape
daily_death_deltas.head()
daily_death_deltas.to_csv('daily_death_deltas.csv', index=False )
def run() :
global_models = {}
local_models = {}
for model_name in models:
DELG_MODEL = tf.saved_model.load(
 model_config[model_name]["SAVED_MODEL_DIR"])
 global_models[model_name] = get_global_extractor(
 DELG_MODEL, model_config[model_name]["is_tf2"])
 if not model_config[model_name]["global_only"]:
local_models[model_name] = get_local_extractor(DELG_MODEL)
 del DELG_MODEL
gc.collect()
tf.keras.backend.clear_session()
labelmap = load_labelmap()
num_training_images = len(labelmap.keys())
 print(f'Found {num_training_images} training images.')
 if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:
print(
 f'Found {NUM_PUBLIC_TRAIN_IMAGES} training images.Copying sample submission.' 
)
 save_submission_csv()
else:
_, post_verification_predictions = get_predictions(
 labelmap, global_models, local_models)
 save_submission_csv(post_verification_predictions)
 del labelmap, global_models, local_models
gc.collect()
tf.keras.backend.clear_session() 
<define_variables>
death_deltas = train.groupby(['Location', 'Country_Region', 'continent'])[[
'Id']].count().reset_index()
GLOBAL_DELTA = 0.11
death_deltas['DELTA'] = GLOBAL_DELTA
death_deltas.loc[death_deltas.Country_Region=='China', 'DELTA'] = 0.005
death_deltas.loc[death_deltas.continent=='Oceania', 'DELTA'] = 0.08
death_deltas.loc[death_deltas.Country_Region=='Korea South', 'DELTA'] = 0.04
death_deltas.loc[death_deltas.Country_Region=='Japan', 'DELTA'] = 0.04
death_deltas.loc[death_deltas.Country_Region=='Singapore', 'DELTA'] = 0.05
death_deltas.loc[death_deltas.Country_Region=='Taiwan*', 'DELTA'] = 0.06
death_deltas.loc[death_deltas.Country_Region=='US', 'DELTA'] = 0.17
death_deltas.loc[death_deltas.Country_Region=='Switzerland', 'DELTA'] = 0.15
death_deltas.loc[death_deltas.Country_Region=='Norway', 'DELTA'] = 0.15
death_deltas.loc[death_deltas.Country_Region=='Iceland', 'DELTA'] = 0.01
death_deltas.loc[death_deltas.Country_Region=='Austria', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Italy', 'DELTA'] = 0.07
death_deltas.loc[death_deltas.Country_Region=='Spain', 'DELTA'] = 0.1
death_deltas.loc[death_deltas.Country_Region=='Portugal', 'DELTA'] = 0.13
death_deltas.loc[death_deltas.Country_Region=='Israel', 'DELTA'] = 0.16
death_deltas.loc[death_deltas.Country_Region=='Iran', 'DELTA'] = 0.06
death_deltas.loc[death_deltas.Country_Region=='Germany', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Malaysia', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Russia', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Ukraine', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Brazil', 'DELTA'] = 0.2
death_deltas.loc[death_deltas.Country_Region=='Turkey', 'DELTA'] = 0.22
death_deltas.loc[death_deltas.Country_Region=='Philippines', 'DELTA'] = 0.12
death_deltas.loc[death_deltas.Location=='France-', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Location=='United Kingdom-', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Location=='Diamond Princess-', 'DELTA'] = 0.00
death_deltas.loc[death_deltas.Location=='China-Hong Kong', 'DELTA'] = 0.01
death_deltas.loc[death_deltas.Location=='San Marino-', 'DELTA'] = 0.05
death_deltas.shape
death_deltas.DELTA.mean()
death_deltas[death_deltas.DELTA != GLOBAL_DELTA].shape
death_deltas[death_deltas.DELTA != GLOBAL_DELTA].DELTA.mean()
death_deltas[death_deltas.DELTA != GLOBAL_DELTA]
death_deltas.describe()
DEBUG = False 
<install_modules>
daily_log_deaths = train_clean.pivot('Location', 'Date', 'LogFatalities' ).reset_index()
daily_log_deaths = daily_log_deaths.sort_values(TRAIN_END, ascending=False)
 daily_log_deaths.to_csv('daily_log_deaths.csv', index=False)
 for i, d in tqdm(enumerate(pd.date_range(add_days(TRAIN_END, 1), add_days(TEST_END, 1)))) :
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in death_deltas.Location:
death_delta = death_deltas.loc[death_deltas.Location == loc, 'DELTA'].values[0]
daily_log_deaths.loc[daily_log_deaths.Location == loc, new_day] = daily_log_deaths.loc[daily_log_deaths.Location == loc, last_day] + \
death_delta * DECAY ** i
!pip install -q.. /input/whl-files/Keras_Applications-1.0.8-py3-none-any.whl
!pip install -q.. /input/whl-files/efficientnet-1.1.0-py3-none-any.whl 
<define_variables>
confirmed = []
fatalities = []
for id, d, loc in tqdm(test[['ForecastId', 'Date', 'Location']].values):
c = to_exp(daily_log_confirmed.loc[daily_log_confirmed.Location == loc, d].values[0])
 f = to_exp(daily_log_deaths.loc[daily_log_deaths.Location == loc, d].values[0])
 confirmed.append(c)
 fatalities.append(f )
IMAGE_SIZE = [512, 512]
NUMBER_OF_CLASSES = 81313
LR = 1e-6
DIM = IMAGE_SIZE[0]
EFNET = 5
QUALITY = 100
MARGIN = 0.3
EMBEDDING_DIMENSION = 2048
NUM_TO_RERANK = 3
TOP_K = 3
if DEBUG:
NUM_PUBLIC_TRAIN_IMAGES = -1
else:
NUM_PUBLIC_TRAIN_IMAGES = 1580470
DATASET_DIR = '.. /input/landmark-recognition-2020'
TRAIN_IMAGE_DIR = '.. /input/landmark-recognition-2020/train'
TEST_IMAGE_DIR = '.. /input/landmark-recognition-2020/test'
TRAIN_LABELMAP_PATH = '.. /input/landmark-recognition-2020/train.csv'
MODEL_PATH = '.. /input/tpu-train-final-dataset/fold-0_epoch-3_valloss-1.3284_loss-0.0056_margin-0.10_scale-64_logweight.h5'<normalization>
my_submission = test.copy()
my_submission['ConfirmedCases'] = confirmed
my_submission['Fatalities'] = fatalities
my_submission.shape
my_submission.head()
class ArcMarginProduct(tf.keras.layers.Layer):
def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,
ls_eps=0.0, **kwargs):
super(ArcMarginProduct, self ).__init__(**kwargs)
 self.n_classes = n_classes
self.s = s
self.m = m
self.ls_eps = ls_eps
self.easy_margin = easy_margin
self.cos_m = tf.math.cos(m)
 self.sin_m = tf.math.sin(m)
 self.th = tf.math.cos(math.pi - m)
 self.mm = tf.math.sin(math.pi - m)* m
def get_config(self):
config = super().get_config().copy()
config.update({
'n_classes': self.n_classes,
's': self.s,
'm': self.m,
'ls_eps': self.ls_eps,
'easy_margin': self.easy_margin,
})
 return config
def build(self, input_shape):
super(ArcMarginProduct, self ).build(input_shape[0])
 self.W = self.add_weight(
 name='W',
shape=(int(input_shape[0][-1]), self.n_classes),
initializer='glorot_uniform',
dtype='float32',
trainable=True,
regularizer=None)
 def call(self, inputs):
X, y = inputs
y = tf.cast(y, dtype=tf.int32)
 cosine = tf.matmul(
 tf.math.l2_normalize(X, axis=1),
tf.math.l2_normalize(self.W, axis=0)
)
 sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))
phi = cosine * self.cos_m - sine * self.sin_m
if self.easy_margin:
phi = tf.where(cosine > 0, phi, cosine)
 else:
phi = tf.where(cosine > self.th, phi, cosine - self.mm)
 one_hot = tf.cast(
 tf.one_hot(y, depth=self.n_classes),
dtype=cosine.dtype 
)
 if self.ls_eps > 0:
one_hot =(1 - self.ls_eps)* one_hot + self.ls_eps / self.n_classes
output =(one_hot * phi)+(( 1.0 - one_hot)* cosine)
 output *= self.s
return output
EFNS = [efn.EfficientNetB0, efn.EfficientNetB1, efn.EfficientNetB2, efn.EfficientNetB3,
efn.EfficientNetB4, efn.EfficientNetB5, efn.EfficientNetB6, efn.EfficientNetB7]
def get_model(ef):
margin = ArcMarginProduct(
 n_classes = NUMBER_OF_CLASSES,
s = 64,
m = 0.1,
name='head/arc_margin',
dtype='float32' 
)
 inp = tf.keras.layers.Input(shape =(*IMAGE_SIZE, 3), name = 'inp1')
 label = tf.keras.layers.Input(shape =() , name = 'inp2')
 x = EFNS[ef](weights = None, include_top = False )(inp)
 x = tf.keras.layers.GlobalAveragePooling2D()(x)
 x = margin([x, label])
 output = tf.keras.layers.Softmax(dtype='float32' )(x)
 model = tf.keras.models.Model(inputs = [inp, label], outputs = [output])
 opt = tf.keras.optimizers.Adam(learning_rate = LR)
 model.compile(
 optimizer = opt,
loss = [tf.keras.losses.SparseCategoricalCrossentropy() ],
metrics = [tf.keras.metrics.SparseCategoricalAccuracy() ] 
)
 return model<load_pretrained>
my_submission[[
'ForecastId', 'ConfirmedCases', 'Fatalities'
]].to_csv('submission.csv', index=False)
 print(DECAY)
 my_submission.head()
my_submission.tail()
my_submission.shape
<define_variables><EOS> 
end = dt.datetime.now()
print('Finished', end,(end - start ).seconds, 's' )