Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<load_from_csv>
!pip install pyramid-arima
def main() :
labelmap = load_labelmap()
num_training_images = len(labelmap.keys())
 print(f'Found {num_training_images} training images')
 if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:
print(f'Found {NUM_PUBLIC_TRAIN_IMAGES} training images.Copying sample submission')
 save_submission_csv()
return
pre_verification_predictions, post_verification_predictions = get_predictions(labelmap)
 save_submission_csv(post_verification_predictions)
 print('Done')
 main()<import_modules>
warnings.filterwarnings("ignore")
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 train.head()
INPUT_DIR = os.path.join('.. ', 'input')
 DATASET_DIR = os.path.join(INPUT_DIR, 'landmark-recognition-2020')
 TEST_IMAGE_DIR = os.path.join(DATASET_DIR, 'test')
 TRAIN_IMAGE_DIR = os.path.join(DATASET_DIR, 'train')
 TRAIN_LABELMAP_PATH = os.path.join(DATASET_DIR, 'train.csv')
 NUM_PUBLIC_TRAIN_IMAGES = 1580470
MAX_NUM_EMBEDDINGS = -1
NUM_TO_RERANK = 6
TOP_K = 3
MAX_INLIER_SCORE = 26
MAX_REPROJECTION_ERROR = 6.0
MAX_RANSAC_ITERATIONS = 900000
HOMOGRAPHY_CONFIDENCE = 0.95
SAVED_MODEL_DIR = '.. /input/delg-saved-models/local_and_global'
DELG_MODEL = tf.saved_model.load(SAVED_MODEL_DIR)
 DELG_IMAGE_SCALES_TENSOR = tf.convert_to_tensor([0.70710677, 1.0, 1.4142135])
 DELG_SCORE_THRESHOLD_TENSOR = tf.constant(175.)
DELG_INPUT_TENSOR_NAMES = [
'input_image:0', 'input_scales:0', 'input_abs_thres:0'
]
NUM_EMBEDDING_DIMENSIONS = 2048
GLOBAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(DELG_INPUT_TENSOR_NAMES,
['global_descriptors:0'])
 LOCAL_FEATURE_NUM_TENSOR = tf.constant(1000)
 LOCAL_FEATURE_EXTRACTION_FN = DELG_MODEL.prune(
 DELG_INPUT_TENSOR_NAMES + ['input_max_feature_num:0'],
['boxes:0', 'features:0'])
 def to_hex(image_id)-> str:
return '{0:0{1}x}'.format(image_id, 16)
 def get_image_path(subset, image_id):
name = to_hex(image_id)
 return os.path.join(DATASET_DIR, subset, name[0], name[1], name[2],
'{}.jpg'.format(name))
def load_image_tensor(image_path):
return tf.convert_to_tensor(
 np.array(PIL.Image.open(image_path ).convert('RGB')))
 def extract_global_features(image_root_dir):

 image_paths = [x for x in pathlib.Path(image_root_dir ).rglob('*.jpg')]
num_embeddings = len(image_paths)
 if MAX_NUM_EMBEDDINGS > 0:
num_embeddings = min(MAX_NUM_EMBEDDINGS, num_embeddings)
 ids = num_embeddings * [None]
embeddings = np.empty(( num_embeddings, NUM_EMBEDDING_DIMENSIONS))
for i, image_path in enumerate(image_paths):
if i >= num_embeddings:
break
ids[i] = int(image_path.name.split('.')[0], 16)
 image_tensor = load_image_tensor(image_path)
 features = GLOBAL_FEATURE_EXTRACTION_FN(image_tensor,
DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR)
 embeddings[i, :] = tf.nn.l2_normalize(
 tf.reduce_sum(features[0], axis=0, name='sum_pooling'),
axis=0,
name='final_l2_normalization' ).numpy()
return ids, embeddings
def extract_local_features(image_path):

 image_tensor = load_image_tensor(image_path)
 features = LOCAL_FEATURE_EXTRACTION_FN(image_tensor, DELG_IMAGE_SCALES_TENSOR,
DELG_SCORE_THRESHOLD_TENSOR,
LOCAL_FEATURE_NUM_TENSOR)
 keypoints = tf.divide(
 tf.add(
 tf.gather(features[0], [0, 1], axis=1),
tf.gather(features[0], [2, 3], axis=1)) , 2.0 ).numpy()
descriptors = tf.nn.l2_normalize(
 features[1], axis=1, name='l2_normalization' ).numpy()
return keypoints, descriptors
def get_putative_matching_keypoints(test_keypoints,
test_descriptors,
train_keypoints,
train_descriptors,
max_distance=0.9):

 train_descriptor_tree = spatial.cKDTree(train_descriptors)
 _, matches = train_descriptor_tree.query(
 test_descriptors, distance_upper_bound=max_distance)
 test_kp_count = test_keypoints.shape[0]
train_kp_count = train_keypoints.shape[0]
test_matching_keypoints = np.array([
test_keypoints[i,]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 train_matching_keypoints = np.array([
train_keypoints[matches[i],]
for i in range(test_kp_count)
 if matches[i] != train_kp_count
])
 return test_matching_keypoints, train_matching_keypoints
def get_num_inliers(test_keypoints, test_descriptors, train_keypoints,
train_descriptors):

 test_match_kp, train_match_kp = get_putative_matching_keypoints(
 test_keypoints, test_descriptors, train_keypoints, train_descriptors)
 if test_match_kp.shape[
0] <= 4:
return 0
try:
_, mask = pydegensac.findHomography(test_match_kp, train_match_kp,
MAX_REPROJECTION_ERROR,
HOMOGRAPHY_CONFIDENCE,
MAX_RANSAC_ITERATIONS)
 except np.linalg.LinAlgError:
return 0
return int(copy.deepcopy(mask ).astype(np.float32 ).sum())
 def get_total_score(num_inliers, global_score):
local_score = min(num_inliers, MAX_INLIER_SCORE)/ MAX_INLIER_SCORE
return local_score + global_score
def rescore_and_rerank_by_num_inliers(test_image_id,
train_ids_labels_and_scores):

 test_image_path = get_image_path('test', test_image_id)
 test_keypoints, test_descriptors = extract_local_features(test_image_path)
 for i in range(len(train_ids_labels_and_scores)) :
train_image_id, label, global_score = train_ids_labels_and_scores[i]
train_image_path = get_image_path('train', train_image_id)
 train_keypoints, train_descriptors = extract_local_features(
 train_image_path)
 num_inliers = get_num_inliers(test_keypoints, test_descriptors,
train_keypoints, train_descriptors)
 total_score = get_total_score(num_inliers, global_score)
 train_ids_labels_and_scores[i] =(train_image_id, label, total_score)
 train_ids_labels_and_scores.sort(key=lambda x: x[2], reverse=True)
 return train_ids_labels_and_scores
def load_labelmap() :
with open(TRAIN_LABELMAP_PATH, mode='r')as csv_file:
csv_reader = csv.DictReader(csv_file)
 labelmap = {row['id']: row['landmark_id'] for row in csv_reader}
return labelmap
def get_prediction_map(test_ids, train_ids_labels_and_scores):

 prediction_map = dict()
for test_index, test_id in enumerate(test_ids):
hex_test_id = to_hex(test_id)
 aggregate_scores = {}
for _, label, score in train_ids_labels_and_scores[test_index][:TOP_K]:
if label not in aggregate_scores:
aggregate_scores[label] = 0
aggregate_scores[label] += score
label, score = max(aggregate_scores.items() , key=operator.itemgetter(1))
prediction_map[hex_test_id] = {'score': score, 'class': label}
return prediction_map
def get_predictions(labelmap):

 test_ids, test_embeddings = extract_global_features(TEST_IMAGE_DIR)
 train_ids, train_embeddings = extract_global_features(TRAIN_IMAGE_DIR)
 train_ids_labels_and_scores = [None] * test_embeddings.shape[0]
for test_index in range(test_embeddings.shape[0]):
distances = spatial.distance.cdist(
 test_embeddings[np.newaxis, test_index, :], train_embeddings,
'cosine')[0]
partition = np.argpartition(distances, NUM_TO_RERANK)[:NUM_TO_RERANK]
nearest = sorted([(train_ids[p], distances[p])for p in partition],
key=lambda x: x[1])
 train_ids_labels_and_scores[test_index] = [ 
(train_id, labelmap[to_hex(train_id)], 1.- cosine_distance)
 for train_id, cosine_distance in nearest
]
del test_embeddings
del train_embeddings
del labelmap
gc.collect()
pre_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 for test_index, test_id in enumerate(test_ids):
train_ids_labels_and_scores[test_index] = rescore_and_rerank_by_num_inliers(
 test_id, train_ids_labels_and_scores[test_index])
 post_verification_predictions = get_prediction_map(
 test_ids, train_ids_labels_and_scores)
 return pre_verification_predictions, post_verification_predictions
def save_submission_csv(predictions=None):

 if predictions is None:
shutil.copyfile(
 os.path.join(DATASET_DIR, 'sample_submission.csv'), 'submission.csv')
 return
with open('submission.csv', 'w')as submission_csv:
csv_writer = csv.DictWriter(submission_csv, fieldnames=['id', 'landmarks'])
 csv_writer.writeheader()
for image_id, prediction in predictions.items() :
label = prediction['class']
score = prediction['score']
csv_writer.writerow({'id': image_id, 'landmarks': f'{label} {score}'})
 def main() :
labelmap = load_labelmap()
num_training_images = len(labelmap.keys())
 print(f'Found {num_training_images} training images.')
 if num_training_images == NUM_PUBLIC_TRAIN_IMAGES:
print(
 f'Found {NUM_PUBLIC_TRAIN_IMAGES} training images.Copying sample submission.' 
)
 save_submission_csv()
return
_, post_verification_predictions = get_predictions(labelmap)
 save_submission_csv(post_verification_predictions)
 if __name__ == '__main__':
main()<set_options>
no_countr = train['Country_Region'].nunique()
no_province = train['Province_State'].nunique()
no_countr_with_prov = len(train[train['Province_State'].isna() ==False]['Country_Region'].unique())
 total_forecasting_number = no_province + no_countr - no_countr_with_prov+2
no_days = train['Date'].nunique()
print('there are ', no_countr, 'unique Countries/Regionions, each with ', no_days, 'days of data, all of them having the same dates.There are also ',no_province, 'Provinces/States which can be found on ', no_countr_with_prov, 'countries/ regions.' )
plt.rcdefaults()
plt.style.use('dark_background')
 warnings.filterwarnings("ignore" )<define_variables>
df = confirmed_total_date.copy()
df = pd.DataFrame({'date': [df.index[i] for i in range(len(df)) ] , 'cases': df['ConfirmedCases'].values.reshape(1,-1)[0].tolist() })
 dfog = df.copy()
def l_regr(x,y):
model = LinearRegression().fit(x, y)
 return model
x = df['cases']
x = x.drop(x.index[-1] ).values.reshape(( -1, 1))
y = df['cases']
y = y.drop(y.index[0])
 ex_slope = l_regr(x,y ).coef_
d = 0
for i in range(1,5):
plt.plot(df['cases'])
 plt.show()
plt.close()
df['prev_cases'] = df['cases'].shift(1)
 df['cases'] =(df['cases'] - df['prev_cases'])
 df = df.drop(['prev_cases'],axis=1)
 df = df.drop(df.index[0])
 x = df['cases']
x = x.drop(y.index[-1] ).values.reshape(( -1, 1))
y = df['cases']
y = y.drop(y.index[0])
 model = l_regr(x,y)
 if(abs(model.coef_)> ex_slope):
print('this is it! ', ex_slope)
 break
d += 1
ex_slope = model.coef_
print(model.coef_ )
INPUT_PATH = os.path.join('.. ', 'input')
 DATASET_PATH = os.path.join(INPUT_PATH, 'landmark-recognition-2020')
 TRAIN_IMAGE_PATH = os.path.join(DATASET_PATH, 'train')
 TEST_IMAGE_PATH = os.path.join(DATASET_PATH, 'test')
 TRAIN_CSV_PATH = os.path.join(DATASET_PATH, 'train.csv')
 SUBMISSION_CSV_PATH = os.path.join(DATASET_PATH, 'sample_submission.csv' )<load_from_csv>
index = 1
cases_pred= []
fatalities_pred = []
pbar = tqdm(total=total_forecasting_number)
 while index < total_forecasting_number+3:
x = train['ConfirmedCases'].iloc[[i for i in range(no_days*(index-1),no_days*index)]].values
z = train['Fatalities'].iloc[[i for i in range(no_days*(index-1),no_days*index)]].values
index += 1
no_nul_cases = pd.DataFrame(x)
 no_nul_cases = no_nul_cases[no_nul_cases.values != 0]
if(not no_nul_cases.empty):
X = [xi for xi in no_nul_cases.values]
try:
model = pyramid.arima.auto_arima(X,seasonal=True, m=12)
 pred = model.predict(31)
 pred = pred.astype(int)
 pred = pred.tolist()
except:
model = l_regr(np.array([i for i in range(len(X)) ] ).reshape(-1, 1),X)
 pred = [(model.coef_*(len(X)+i)+ model.intercept_ ).astype('int')[0][0] for i in range(1,32)]
else:
pred = [0] * 31
pred = x[-12:].astype(int ).tolist() + pred
cases_pred+=pred
no_nul_fatalities = pd.DataFrame(z)
 no_nul_fatalities = no_nul_fatalities[no_nul_fatalities.values != 0]
if(not no_nul_fatalities.empty):
Z = [zi for zi in no_nul_fatalities.values]
try:
model = pyramid.arima.auto_arima(Z, seasonal=False, m=12)
 pred = model.predict(31)
 pred = pred.astype(int)
 pred = pred.tolist()
except:
model = l_regr(np.array([i for i in range(len(Z)) ] ).reshape(-1, 1),Z)
 pred = [(model.coef_*(len(Z)+i)+ model.intercept_ ).astype('int')[0][0] for i in range(1,32)]
else:
pred = [0] * 31
pred = z[-12:].astype(int ).tolist() + pred
fatalities_pred+=pred
pbar.update(1)
 pbar.close()
train = pd.read_csv(TRAIN_CSV_PATH)
 print("training dataset has {} rows and {} columns".format(train.shape[0],train.shape[1]))
submission = pd.read_csv(SUBMISSION_CSV_PATH)
 print("submission dataset has {} rows and {} columns
".format(submission.shape[0],submission.shape[1]))<define_variables>
submission = pd.DataFrame({'ForecastId': [i for i in range(1,len(cases_pred)+1)] ,'ConfirmedCases': cases_pred, 'Fatalities': fatalities_pred})
 filename = 'submission.csv'
submission.to_csv(filename,index=False )