Kaggle competition: COVID19 Global Forecasting (Week 4)
id_cols = ["Id","groupId","matchId"]
exclude = ["Id","groupId","matchId"]
trainlen = train.shape[0]
matchcount = train.matchId.nunique()
training = train.loc[train.matchId.isin(sorted(train.matchId.unique())[:int(matchcount* 0.85)]),
[x for x in train.columns if x not in exclude]]
print("Training Shape: {} Rows, {} Cols".format(*training.shape))
validating = train.loc[train.matchId.isin(sorted(train.matchId.unique())[int(matchcount* 0.85):]),
[x for x in train.columns if x not in exclude]]
print("Validating Shape: {} Rows, {} Cols".format(*validating.shape))
train_y = training.winPlacePerc
training.drop("winPlacePerc", axis =1, inplace=True)
 valid_y = validating.winPlacePerc
validating.drop("winPlacePerc", axis =1, inplace=True)
 lgb_train = lgb.Dataset(training, train_y,feature_name = "auto")
 lgb_valid = lgb.Dataset(validating, valid_y, feature_name = "auto")
 del training, validating<init_hyperparams>
with open('/kaggle/input/covid4submissions/week4_ml_30_final.csv', 'rt')as infile:
with open('submission.csv', 'wt')as outfile:
writer = csv.writer(outfile)
 for line in csv.reader(infile):
writer.writerow(line )
print("Light Gradient Boosting Regressor: ")
 lgbm_params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': 'rmse',
'num_boost_round' : 5000
}<save_to_csv>
from collections import defaultdict
import collections
import csv
from datetime import datetime, timedelta
import glob
import logging
from math import *
import numpy as np
import optparse
import os
import time
import sys
import copy
from typing import Any, Dict, List, Optional, Tuple, Union
import xgboost as xgb
from catboost import Pool, CatBoostRegressor
from xml.etree import ElementTree
from scipy.optimize import curve_fit
pred = gbm.predict(test.loc[:,[x for x in test.columns if x not in id_cols]])
 test['winPlacePercPred'] = np.clip(pred, a_min=0, a_max=1)
 aux = test.groupby(['matchId','groupId'])['winPlacePercPred'].agg('mean' ).groupby('matchId' ).rank(pct=True ).reset_index()
aux.columns = ['matchId','groupId','winPlacePerc']
test_sub = test.merge(aux, how='left', on=['matchId','groupId'])
 submission = test_sub[['Id', 'winPlacePerc']]
submission.to_csv('PubGG_LGBM.csv', index=False)
 display(submission.head() )<define_variables>
RUN = False
PROD = True
PROD_DAYS = 30
EVAL_DAYS = 7
notcat = ["assists","boosts","damageDealt","DBNOs","heals","headshotKills","heals","killPlace","killPoints","kills",
"killStreaks","longestKill","maxPlace","numGroups","revives","rideDistance","roadKills","swimDistance",
"teamKills","vehicleDestroys","walkDistance","weaponsAcquired","winPoints"]<prepare_x_and_y>
def read_learn_datasets(path, days):
examples = []
for i in range(days):
examples.extend(read_dataset(path + '_' + str(i+1)+ '.csv'))
return examples
def get_pool_data(mode, data, prefix):
def get(e):
return e if e else 0.0
x, delta_y, abs_y, fit, prev = [], [], [], [], []
for e in data[prefix + 'examples']:
if mode == 'cases':
cur_prev, cur_fit, cur_y = e.features['F_prev_cases_1'], e.features['F_fit_20'], get(e.cases)
 else:
cur_prev, cur_fit, cur_y = e.features['F_prev_fatal_1'], e.features['F_fit_fatal_20'], get(e.fatalities)
 cur_y = math.log(1 + cur_y)
 abs_y.append(cur_y)
 delta_y.append(cur_y - cur_prev)
 x.append(list(e.features.values()))
fit.append(cur_fit)
 prev.append(cur_prev)
 data['names'] = list(e.features.keys())
 data[prefix + 'x'] = np.asarray(x)
 data[prefix + 'delta_y'] = np.asarray(delta_y)
 data[prefix + 'abs_y'] = np.asarray(abs_y)
 data[prefix + 'fit'] = fit
data[prefix + 'prev'] = prev
def read_everything(opts):
data = {}
data['train_examples'] = read_learn_datasets(opts.train_features_csv, opts.days)
 data['test_examples'] = read_learn_datasets(opts.test_features_csv, opts.days)
 get_pool_data(opts.mode, data, 'train_')
 get_pool_data(opts.mode, data, 'test_')
 return data
def get_pool(data, prefix, is_delta, is_catboost):
y_key = prefix + 'delta_y' if is_delta else prefix + 'abs_y'
if is_catboost:
return Pool(data[prefix + 'x'], data[y_key])
 return xgb.DMatrix(data[prefix + 'x'], label=data[y_key])
 def write_predictions(path, data, mode, prefix, predicted, is_delta, name):
with open('_'.join([path, name, mode])+ '.csv', 'w')as f:
writer = csv.writer(f, lineterminator='
', delimiter=',')
 writer.writerow(['key', 'state', 'country', 'date', 'y', 'prediction', 'fit20', 'prev'])
 examples, y, fit, prev = data[prefix + 'examples'], data[prefix + 'abs_y'], data[prefix + 'fit'], data[prefix + 'prev']
for i in range(len(examples)) :
e = examples[i]
pred = predicted[i]
if is_delta:
pred += prev[i]
writer.writerow([e.key, e.state, e.country, e.date, y[i], max(pred, 0), fit[i], prev[i]])
 def model_name(is_delta, is_linear, is_catboost):
prefix = 'delta' if is_delta else 'abs'
if is_linear: return prefix + '_linear'
if is_catboost: return prefix + '_catboost'
return prefix + '_trees'
def do_learn(data, opts, is_delta, is_linear, is_catboost, iterations):
train = get_pool(data, 'train_', is_delta, is_catboost)
 test = get_pool(data, 'test_', is_delta, is_catboost)
 if is_catboost:
return
else:
param = {'nthread' : 16, 'objective' : 'reg:squarederror', 'seed' : opts.seed, 'eta' : opts.eta}
if is_linear:
param['booster'] = 'gblinear'
param['alpha'] = 0.00001
else:
param['max_depth'] = opts.depth
param['subsample'] = 0.8
evallist = [(test, 'eval'),(train, 'train')] if opts.verbose else []
booster = xgb.train(param, train, 2 if opts.super_quick else iterations, evallist)
 train_predicted, test_predicted = booster.predict(train), booster.predict(test)
 write_predictions(opts.train_predicted_csv, data, opts.mode, 'train_', train_predicted, is_delta, model_name(is_delta, is_linear, is_catboost))
write_predictions(opts.test_predicted_csv, data, opts.mode, 'test_', test_predicted, is_delta, model_name(is_delta, is_linear, is_catboost))
def main_LearningStage(opts):
data = read_everything(opts)
 iterations = {}
iterations[('delta', 'trees', 'cases')] = 170
iterations[('delta', 'trees', 'fatal')] = 260
iterations[('abs', 'trees', 'cases')] = 210
iterations[('abs', 'trees', 'fatal')] = 260
iterations[('delta', 'linear', 'cases')] = 4
iterations[('delta', 'linear', 'fatal')] = 10
iterations[('abs', 'linear', 'cases')] = 40
iterations[('abs', 'linear', 'fatal')] = 10
for is_linear in [False, True]:
for is_delta in [True, False]:
key =('delta' if is_delta else 'abs', 'linear' if is_linear else 'trees', opts.mode)
 print('__'.join(['Key'] + list(key)))
 do_learn(data, opts, is_delta, is_linear, False, iterations[key] )
y = train.winPlacePerc.copy()
train.drop("winPlacePerc",axis =1, inplace=True )<merge>
def get_final_score(dt, at, dl, al, weights):
prev = float(dt['prev'])
 dt, at, dl, al = map(lambda x: float(x['prediction']), [dt, at, dl, al])
 pred = 0
pred += weights['wdt'] * dt
pred += weights['wat'] * at
pred += weights['wdl'] * dl
pred += weights['wal'] * al
return max(pred, prev)
 def coeff() :
return [0.0, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.5, 0.75, 0.85, 0.9, 1.0]
def ordered_loc_data(loc_data):
cur = [[k, loc_data[k]] for k in loc_data.keys() ]
cur.sort(key=lambda x: x[0])
 return [x[1] for x in cur]
def get_for_country(weights, values):
error, n = 0.0, 0
for v in values:
sc, sf = v['scores_cases'], v['scores_fatal']
cases = sc[0] * weights['wdt'] + sc[1] * weights['wat'] + sc[2] * weights['wdl'] + sc[3] * weights['wal']
fatal = sf[0] * weights['wdt'] + sf[1] * weights['wat'] + sf[2] * weights['wdl'] + sf[3] * weights['wal']
n += 2
error +=(v['y_cases'] - cases)** 2
fatal +=(v['y_fatal'] - fatal)** 2
return error
def prepare_index(examples_delta_trees, examples_abs_trees, examples_delta_linear, examples_abs_linear, opts, weights, country_weights):
index = {}
for e_dt, e_at, e_dl, e_al in zip(examples_delta_trees, examples_abs_trees, examples_delta_linear, examples_abs_linear):
key = '__'.join(map(str, [e_dt['country'], e_dt['state']]))
if country_weights is None:
pred = get_final_score(e_dt, e_at, e_dl, e_al, weights)
 else:
pred = get_final_score(e_dt, e_at, e_dl, e_al, country_weights[key])
 date = e_dt['date']
if key not in index: index[key] = {}
loc_index = index[key]
if date not in loc_index: loc_index[date] = {}
dct = loc_index[date]
dct['pred_' + e_dt['mode']] = pred
dct['y_' + e_dt['mode']] = float(e_dt['y'])
 dct['obj_' + e_dt['mode']] = e_dt
dct['prev_' + e_dt['mode']] = float(e_dt['prev'])
 dct['scores_' + e_dt['mode']] = list(map(lambda x: float(x['prediction']), [e_dt, e_at, e_dl, e_al]))
for key in index.keys() :
index[key] = ordered_loc_data(index[key])
 return index
def apply_rules(loc_data, min_mortality, max_mortality):
prev_cases = []
prev_fatalities = []
for v in loc_data:
cases, fatalities = v['pred_cases'], v['pred_fatal']
if len(prev_cases)> 0 and cases < prev_cases[-1]:
cases = prev_cases[-1]
if len(prev_fatalities)> 0 and fatalities < prev_fatalities[-1]:
fatalities = prev_fatalities[-1]
if cases - fatalities > min_mortality:
fatalities = cases - min_mortality
if cases - fatalities < max_mortality and cases > 5:
fatalities = cases - max_mortality
v['final_cases'] = max(cases, v['prev_cases'])
 v['final_fatal'] = max(fatalities, v['prev_fatal'])
 prev_cases.append(v['final_cases'])
 prev_fatalities.append(v['final_fatal'])
 def copy_and_set_prediction(e, pred, lst):
tmp = copy.deepcopy(e)
 tmp['prediction'] = pred
lst.append(tmp)
 def calc_error(examples_delta_trees, examples_abs_trees, examples_delta_linear, examples_abs_linear, opts, weights, country_weights):
index = prepare_index(examples_delta_trees, examples_abs_trees, examples_delta_linear, examples_abs_linear, opts, weights, country_weights)
 for loc_data in index.values() :
apply_rules(loc_data, math.log(opts.min_mortality), math.log(opts.max_mortality))
error = 0
cases_list, fatalities_list = [], []
n = 0
for loc_data in index.values() :
for v in loc_data:
if not opts.only_fatal:
n += 1
error +=(v['final_cases'] - v['y_cases'])** 2
if not opts.only_cases:
n += 1
error +=(v['final_fatal'] - v['y_fatal'])** 2
copy_and_set_prediction(v['obj_cases'], v['final_cases'], cases_list)
 copy_and_set_prediction(v['obj_fatal'], v['final_fatal'], fatalities_list)
 return(error / n)** 0.5, cases_list, fatalities_list
def read_examples(pattern, mode):
result = []
with open('{}_{}.csv'.format(pattern, mode), 'r')as f:
reader = csv.DictReader(f, quotechar='"', delimiter=',')
 for row in reader:
row['mode'] = mode
result.append(row)
 return result
def read_examples_for_model(opts, model_name):
test_examples = []
test_examples.extend(read_examples(opts.predicted_pattern + model_name, "cases"))
test_examples.extend(read_examples(opts.predicted_pattern + model_name, "fatal"))
return test_examples
def write_final_predictions(path, lst):
with open(path, 'wt')as f:
w = csv.writer(f)
 w.writerow(lst[0].keys())
 for e in lst:
w.writerow([e[k] for k in e.keys() ])
 def print_error(error, opts, w, win):
s = 'Test error = {}, Win = {}.min_mortality={}, max_mortality={}, formula = {} * DT + {} * AT + {} * DL + {} * AL'
return s.format(error, win, opts.min_mortality, opts.max_mortality, w['wdt'], w['wat'], w['wdl'], w['wal'])
 def main_FinalizingStage(opts):
examples_delta_trees = read_examples_for_model(opts, '_delta_trees')
 examples_abs_trees = read_examples_for_model(opts, '_abs_trees')
 examples_delta_linear = read_examples_for_model(opts, '_delta_linear')
 examples_abs_linear = read_examples_for_model(opts, '_abs_linear')
 country_weights = None
weights = {'wdt' : opts.wdt, 'wat' : opts.wat, 'wdl' : opts.wdl, 'wal' : opts.wal}
prod_error, cases_list, fatalities_list = calc_error(examples_delta_trees, examples_abs_trees, examples_delta_linear, examples_abs_linear, opts, weights, country_weights)
 print(prod_error)
 write_final_predictions(opts.predicted_pattern + '_cases.csv', cases_list)
 write_final_predictions(opts.predicted_pattern + '_fatalities.csv', fatalities_list)
def agg_dataset(dataset):
for id_col in id_cols:
agg_features = dataset.groupby(id_col ).agg({k:["sum","mean","std"] for k in
["killPlace","walkDistance","numGroups","maxPlace","kills","longestKill","weaponsAcquired"]})
 agg_features.columns = pd.Index(["{}_agg_".format(id_col)+ e[0] +"_"+ e[1] for e in agg_features.columns.tolist() ])
 dataset = pd.merge(dataset,agg_features, on = id_col, how= "left")
 return dataset<create_dataframe>
def main_Week2Adjustment(opts):
data = {}
with open(opts.train, 'r')as f:
reader = csv.DictReader(f, quotechar='"', delimiter=',')
 for row in reader:
key = '__'.join([row['Province_State'], row['Country_Region']])
 if key not in data: data[key] = [-1, -1, -1, -1]
if row['Date'] == '2020-03-27':
data[key][0] = math.log(1 + float(row['ConfirmedCases']))
data[key][1] = math.log(1 + float(row['Fatalities']))
if row['Date'] == '2020-03-31':
data[key][2] = math.log(1 + float(row['ConfirmedCases']))
data[key][3] = math.log(1 + float(row['Fatalities']))
hack = {}
with open(opts.cases_predicted_csv, 'r')as f:
writer = csv.writer(open(opts.modified_cases, 'wt'))
reader = csv.DictReader(f, quotechar='"', delimiter=',')
 last_country = 'None'
writer.writerow(['key','state','country','date','prediction', 'fit20'])
 for row in reader:
key = '__'.join([row['state'], row['country']])
 val = data[key]
if last_country != key:
last_country = key
last_value = data[key][2]
delta =(val[2] - val[0])/ 4
delta = max(0, delta)
 else:
delta *= 0.925
pred = float(row['prediction'])
 at_most = last_value + delta
pred = min(pred, at_most)
 last_value = pred
hack['__'.join([row['state'], row['country'], row['date']])] = pred
writer.writerow([row['key'], row['state'], row['country'], row['date'], pred, 0])
 with open(opts.fatal_predicted_csv, 'r')as f:
writer = csv.writer(open(opts.modified_fatal, 'wt'))
reader = csv.DictReader(f, quotechar='"', delimiter=',')
 last_country = 'None'
writer.writerow(['key','state','country','date','prediction', 'fit20'])
 for row in reader:
key = '__'.join([row['state'], row['country']])
 val = data[key]
if last_country != key:
last_country = key
last_value = data[key][3]
delta =(val[2] - val[0])/ 4
delta = max(0, delta)
 else:
delta *= 0.93
pred = float(row['prediction'])
 at_most = last_value + delta
pred = min(pred, at_most)
 cases_value = hack['__'.join([row['state'], row['country'], row['date']])]
if pred < cases_value - opts.max_mortality_log:
pred = cases_value - opts.max_mortality_log
last_value = pred
writer.writerow([row['key'], row['state'], row['country'], row['date'], pred, 0] )
train = agg_dataset(train)
 test = agg_dataset(test )<drop_column>
def read_one(path):
res = []
with open(path, 'r')as f:
for row in csv.DictReader(f, quotechar='"', delimiter=','):
res.append(row)
 return res
def main_Submission(opts):
submission = read_one(opts.submission)
 cases = read_one(opts.predicted_cases)
 fatal = read_one(opts.predicted_fatalities)
 cases_index = {}
for case in cases:
cases_index[case['state'] + '__' + case['country'] + '__' + case['date']] = math.exp(float(case['prediction'])) - 1
fatal_index = {}
for case in fatal:
fatal_index[case['state'] + '__' + case['country'] + '__' + case['date']] = math.exp(float(case['prediction'])) - 1
n, found = 0, 0
with open(opts.output, 'w')as f:
writer = csv.writer(f, lineterminator='
', delimiter=',')
 writer.writerow(['ForecastId', 'ConfirmedCases', 'Fatalities'])
 for rec in submission:
key = rec['Province_State'] + '__' + rec['Country_Region'] + '__' + rec['Date']
cases_answer, fatal_answer = 0, 0
n += 1
if key in cases_index:
found += 1
cases_answer, fatal_answer = cases_index[key], fatal_index[key]
writer.writerow([rec['ForecastId'], cases_answer, fatal_answer])
 print('N={}'.format(n))
print('Found={}'.format(found))
print('Share={}'.format(float(found)/ n))
missing = round(train.isnull().sum() / train.shape[0]*100 ).reset_index().rename({"index":"columns",0:"missing"}, axis =1)
 high_missing_columns = missing.loc[missing.missing > 65, "columns"]
print("Columns to remove(65% missing Values and Over)
 ", list(high_missing_columns))
train.drop(high_missing_columns,axis =1, inplace= True)
 test.drop(high_missing_columns,axis =1, inplace= True )<prepare_x_and_y>
class FeaturesOpts:
def __init__(self, num_days, predict_window, verbose, start_date, last_n_days = -1):
self.train_csv = '/kaggle/input/covid19-global-forecasting-week-2/train.csv'
self.test_csv = '/kaggle/input/covid19-global-forecasting-week-2/test.csv'
self.train_features_csv = 'train_features_{}.csv'.format(predict_window)
 self.test_features_csv = 'test_features_{}.csv'.format(predict_window)
 self.num_days = num_days
self.predict_window = predict_window
self.verbose = verbose
self.start_date = start_date
self.last_n_days = last_n_days
start = time.time()
if RUN:
if PROD:
for i in range(PROD_DAYS):
print('Day ' + str(i + 1))
main_FeaturesStage(FeaturesOpts(PROD_DAYS, i + 1, False, '2020-04-02'))
else:
for i in range(EVAL_DAYS):
print('Day ' + str(i + 1))
main_FeaturesStage(FeaturesOpts(EVAL_DAYS, i + 1, False, '2020-03-31'))
end = time.time()
print('Overall took {} minutes'.format(( end - start)/ 60))
train = pd.concat([train.reset_index(drop=True), y.reset_index(drop=True)], axis=1)
 training = train.loc[train.matchId.isin(sorted(train.matchId.unique())[:int(matchcount* 0.85)]),
[x for x in train.columns if x not in exclude]]
print("Training Shape: {} Rows, {} Cols".format(*training.shape))
validating = train.loc[train.matchId.isin(sorted(train.matchId.unique())[int(matchcount* 0.85):]),
[x for x in train.columns if x not in exclude]]
print("Validating Shape: {} Rows, {} Cols".format(*validating.shape))
train_y = training.winPlacePerc
training.drop("winPlacePerc", axis =1, inplace=True)
 valid_y = validating.winPlacePerc
validating.drop("winPlacePerc", axis =1, inplace=True)
 lgb_train = lgb.Dataset(training, train_y,feature_name = "auto")
 lgb_valid = lgb.Dataset(validating, valid_y, feature_name = "auto")
<init_hyperparams>
class LearningOpts:
def __init__(self, num_days, mode, super_quick = False):
self.train_features_csv = 'train_features'
self.test_features_csv = 'test_features'
self.train_predicted_csv = 'train_predicted'
self.test_predicted_csv = 'test_predicted'
self.mode = mode
self.days = num_days
self.verbose = True
self.eta = 0.03
self.seed = 0
self.depth = 6
self.super_quick = super_quick
start = time.time()
if RUN:
if PROD:
main_LearningStage(LearningOpts(PROD_DAYS, 'cases'))
main_LearningStage(LearningOpts(PROD_DAYS, 'fatal'))
else:
main_LearningStage(LearningOpts(EVAL_DAYS, 'cases', False))
main_LearningStage(LearningOpts(EVAL_DAYS, 'fatal'))
end = time.time()
print('Overall took {} minutes'.format(( end - start)/ 60))
print("Light Gradient Boosting Regressor: ")
 lgbm_params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': 'rmse',
'num_boost_round' : 5000
}
stage = 'model training'
gbm = lgb.train(lgbm_params,
lgb_train,
num_boost_round=10000,
valid_sets=[lgb_train, lgb_valid],
feature_name='auto',
early_stopping_rounds=50,
verbose_eval=250 
)
 f, ax = plt.subplots(figsize=[7,10])
 lgb.plot_importance(gbm, max_num_features=25, ax=ax)
 plt.title("Light GBM Feature Importance
")
 plt.show()<save_to_csv>
class FinalizingOpts:
def __init__(self):
self.predicted_pattern = 'test_predicted'
self.verbose = True
self.only_cases = False
self.only_fatal = False
self.wdt = 0.5
self.wat = 0.3
self.wdl = 0.1
self.wal = 0.1
self.min_mortality = 250.0
self.max_mortality = 1.0
self.optimize = False
if RUN:
main_FinalizingStage(FinalizingOpts() )
pred = gbm.predict(test.loc[:,[x for x in test.columns if x not in id_cols]])
 test['winPlacePercPred'] = np.clip(pred, a_min=0, a_max=1)
 aux = test.groupby(['matchId','groupId'])['winPlacePercPred'].agg('mean' ).groupby('matchId' ).rank(pct=True ).reset_index()
aux.columns = ['matchId','groupId','winPlacePerc']
test_sub = test.merge(aux, how='left', on=['matchId','groupId'])
 submission = test_sub[['Id', 'winPlacePerc']]
submission.to_csv('AGG_PubGG_LGBM.csv', index=False )<import_modules>
class Week2Opts:
def __init__(self):
self.train = '/kaggle/input/covid19-global-forecasting-week-2/train.csv'
self.cases_predicted_csv = 'test_predicted_cases.csv'
self.fatal_predicted_csv = 'test_predicted_fatalities.csv'
self.modified_cases = 'modified_cases.csv'
self.modified_fatal = 'modified_fatal.csv'
self.max_mortality_log = 4.7
if RUN and PROD:
main_Week2Adjustment(Week2Opts() )
from xgboost import XGBRegressor<load_from_csv>
class SubmissionOpts:
def __init__(self, use_modified = True):
self.submission = '/kaggle/input/covid19-global-forecasting-week-2/test.csv'
self.predicted_cases = 'modified_cases.csv' if use_modified else 'test_predicted_cases.csv'
self.predicted_fatalities = 'modified_fatal.csv' if use_modified else 'test_predicted_fatalities.csv'
self.output = 'submission.csv'
self.verbose = True
if RUN and PROD:
main_Submission(SubmissionOpts(True))