Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<load_from_csv>
py.init_notebook_mode(connected=True)
 pio.templates.default = "plotly_dark"
train_df = pd.read_csv('.. /input/train.csv', na_values="-1")
 test_df = pd.read_csv('.. /input/test.csv', na_values="-1" )<define_variables>
mat.rcParams.update({'figure.figsize':(20,15),'font.size':14})
 covid19_train = pd.read_csv('.. /input/covid19-global-forecasting-week-2/train.csv')
 covid19_test = pd.read_csv('.. /input/covid19-global-forecasting-week-2/test.csv')
 covid19_train.rename(columns={'Id':'ForecastId'},inplace=True)
 data = pd.read_csv('.. /input/coviddata/time-series-19-covid-combined.csv')
train_features = [
"ps_car_13",
	"ps_reg_03",
	"ps_ind_05_cat",
	"ps_ind_03",
	"ps_ind_15",
	"ps_reg_02",
	"ps_car_14",
	"ps_car_12",
	"ps_car_01_cat",
	"ps_car_07_cat",
	"ps_ind_17_bin",
	"ps_car_03_cat",
	"ps_reg_01",
	"ps_car_15",
	"ps_ind_01",
	"ps_ind_16_bin",
	"ps_ind_07_bin",
	"ps_car_06_cat",
	"ps_car_04_cat",
	"ps_ind_06_bin",
	"ps_car_09_cat",
	"ps_car_02_cat",
	"ps_ind_02_cat",
	"ps_car_11",
	"ps_car_05_cat",
	"ps_calc_09",
	"ps_calc_05",
	"ps_ind_08_bin",
	"ps_car_08_cat",
	"ps_ind_09_bin",
	"ps_ind_04_cat",
	"ps_ind_18_bin",
	"ps_ind_12_bin",
	"ps_ind_14",
]
combs = [ 
('ps_reg_01', 'ps_car_02_cat'), 
('ps_reg_01', 'ps_car_04_cat'),
]<prepare_x_and_y>
covid19_train['Date-All'] = covid19_train['Date'].str.replace('-','' ).astype(int)
 covid19_test['Date-All'] = covid19_test['Date'].str.replace('-','' ).astype(int)
 data['Date-All'] =data['Date'].str.replace('-','' ).astype(int)
 covid19_train['Date'] = pd.to_datetime(covid19_train['Date'])
 covid19_test['Date'] = pd.to_datetime(covid19_test['Date'])
 data['Date'] = pd.to_datetime(data['Date'] )
id_test = test_df['id'].values
id_train = train_df['id'].values
y = train_df['target']
start = time.time()
for n_c,(f1, f2)in enumerate(combs):
name1 = f1 + "_plus_" + f2
print('current feature %60s %4d in %5.1f'
%(name1, n_c + 1,(time.time() - start)/ 60), end='')
 print('\r' * 75, end='')
 train_df[name1] = train_df[f1].apply(lambda x: str(x)) + "_" + train_df[f2].apply(lambda x: str(x))
test_df[name1] = test_df[f1].apply(lambda x: str(x)) + "_" + test_df[f2].apply(lambda x: str(x))
lbl = LabelEncoder()
lbl.fit(list(train_df[name1].values)+ list(test_df[name1].values))
train_df[name1] = lbl.transform(list(train_df[name1].values))
test_df[name1] = lbl.transform(list(test_df[name1].values))
train_features.append(name1)
 X = train_df[train_features]
test_df = test_df[train_features]
f_cats = [f for f in X.columns if "_cat" in f] 
<define_variables>
covid19_gdf = covid19_train.groupby(['Date','Country_Region'])['ConfirmedCases'].sum().reset_index()
covid19_gdf['date'] = pd.to_datetime(covid19_gdf['Date'])
 covid19_gdf['date'] = covid19_gdf['date'].dt.strftime('%m/%d/%Y')
 data_gdf = data.groupby(['Date','Country/Region'])['Confirmed'].sum().reset_index()
data_gdf['date'] = pd.to_datetime(data['Date'])
 data_gdf['date'] = data_gdf['date'].dt.strftime('%m/%d/%Y')
y_valid_pred = 0*y
y_test_pred = 0<choose_model_class>
conf=pd.read_csv(".. /input/coviddata/time_series_covid19_confirmed_global.csv")
 death=pd.read_csv(".. /input/coviddata/time_series_covid19_deaths_global.csv")
 recov=pd.read_csv(".. /input/coviddata/time_series_covid19_recovered_global.csv")
 cum=pd.read_csv(".. /input/coviddata/time_series_covid19_cumudf_global.csv")
 daily=pd.read_csv(".. /input/coviddata/time_series_covid19_dailydf_global.csv")
 cols=conf.keys()
confirmed = conf.loc[:, cols[4]:cols[-1]]
deaths = death.loc[:, cols[4]:cols[-1]]
recoveries = recov.loc[:, cols[4]:cols[-1]]
K = 5
kf = KFold(n_splits = K, random_state = 1, shuffle = True)
 np.random.seed(0 )<choose_model_class>
totalDailyCases = covid19_train.groupby(['Date'])[['ConfirmedCases']].sum().sort_values('ConfirmedCases',ascending=False)
 totalDailyDeaths = covid19_train.groupby(['Date'])[['Fatalities']].sum().sort_values('Fatalities',ascending=False)
xgbmodel = XGBClassifier(
 n_estimators=MAX_XGB_ROUNDS,
max_depth=4,
objective="binary:logistic",
learning_rate=XGB_LEARNING_RATE,
subsample=.8,
min_child_weight=6,
colsample_bytree=.8,
scale_pos_weight=1.6,
gamma=10,
reg_alpha=8,
reg_lambda=1.3, 
)
 rgf = RGFClassifier(
 max_leaf=1200,
algorithm="RGF",
loss="Log",
l2=0.01,
sl2=0.01,
normalize=False,
min_samples_leaf=10,
n_iter=None,
opt_interval=100,
learning_rate=.5,
calc_prob="sigmoid",
n_jobs=-1,
memory_policy="generous",
verbose=0
)<split>
china_cases = []
italy_cases = []
us_cases = []
spain_cases = []
india_cases=[]
days=cum['Days']
n=confirmed.keys()
for i,(train_index, test_index)in enumerate(kf.split(train_df)) :
y_train, y_valid = y.iloc[train_index].copy() , y.iloc[test_index]
X_train, X_valid = X.iloc[train_index,:].copy() , X.iloc[test_index,:].copy()
X_test = test_df.copy()
print("
Fold ", i)
 for f in f_cats:
X_train[f + "_avg"], X_valid[f + "_avg"], X_test[f + "_avg"] = target_encode(
 trn_series=X_train[f],
val_series=X_valid[f],
tst_series=X_test[f],
target=y_train,
min_samples_leaf=200,
smoothing=10,
noise_level=0 
)
 if USE_RGF_INSTEAD:
X_train = X_train.fillna(X_train.mean())
 rgf.fit(X_train, y_train)
 elif OPTIMIZE_XGB_ROUNDS:
eval_set=[(X_valid,y_valid)]
fit_model = xgbmodel.fit(X_train, y_train,
eval_set=eval_set,
eval_metric=gini_xgb,
early_stopping_rounds=XGB_EARLY_STOPPING_ROUNDS,
verbose=False 
)
 print(" Best N trees = ", xgbmodel.best_ntree_limit)
 print(" Best gini = ", xgbmodel.best_score)
 else:
fit_model = xgbmodel.fit(X_train, y_train)
 if USE_RGF_INSTEAD:
pred = rgf.predict_proba(X_valid.fillna(X_train.mean())) [:,1]
else:
pred = fit_model.predict_proba(X_valid)[:,1]
print(" Gini = ", eval_gini(y_valid, pred))
y_valid_pred.iloc[test_index] = pred
if USE_RGF_INSTEAD:
probs = rgf.predict_proba(X_test.fillna(X_train.mean())) [:,1]
try:
subprocess.call('rm -rf /tmp/rgf/*', shell=True)
 print("Clean up is successfull")
 print(glob.glob("/tmp/rgf/*"))
except Exception as e:
print(str(e))
else:
probs = fit_model.predict_proba(X_test)[:,1]
almost_zero = 1e-12
almost_one = 1 - almost_zero
probs[probs>almost_one] = almost_one
probs[probs<almost_zero] = almost_zero
y_test_pred += np.log(probs/(1-probs))
del X_test, X_train, X_valid, y_train
y_test_pred /= K
y_test_pred = 1 /(1 + np.exp(-y_test_pred))
print("
Gini for full training set:")
 eval_gini(y, y_valid_pred )<save_to_csv>
china_cases=np.array(china_cases ).reshape(1,-1)
 italy_cases=np.array(italy_cases ).reshape(1,-1)
 spain_cases=np.array(spain_cases ).reshape(1,-1)
 india_cases=np.array(india_cases ).reshape(1,-1)
 us_cases=np.array(us_cases ).reshape(1,-1)
val = pd.DataFrame()
val['id'] = id_train
val['target'] = y_valid_pred.values
val.to_csv('kfloa_valid.csv', float_format='%.6f', index=False )<save_to_csv>
mort_china=china_death/china_cases
mort_italy=italy_death/italy_cases
mort_spain=spain_death/spain_cases
mort_india=india_death/india_cases
mort_us=us_death/us_cases
sub = pd.DataFrame()
sub['id'] = id_test
sub['target'] = y_test_pred
sub.to_csv('kfloa_submit.csv', float_format='%.6f', index=False )<load_from_csv>
import matplotlib.pyplot as plt
import numpy as np
from scipy import integrate, optimize
<load_from_csv>
def fa(N, a, b, beta):
fa = -beta*a*b
return fa
def fb(N, a, b, beta, gamma):
fb = beta*a*b - gamma*b
return fb
def fc(N, b, gamma):
fc = gamma*b
return fc
dflgbm = pd.read_csv(".. /input/adjustlgbm1003/justSetZeroProbas.csv")
 dflgbm=dflgbm.sort_values('object_id' ).reset_index(drop=True)
 dfnn=pd.read_csv(".. /input/neuralmoredecisive/moreDecisiveNeural.csv")
 dfnn=dfnn.sort_values('object_id' ).reset_index(drop=True)
 print(dflgbm.shape)
 print(dfnn.shape )<feature_engineering>
def rK4(N, a, b, c, fa, fb, fc, beta, gamma, hs):
a1 = fa(N, a, b, beta)*hs
b1 = fb(N, a, b, beta, gamma)*hs
c1 = fc(N, b, gamma)*hs
ak = a + a1*0.5
bk = b + b1*0.5
ck = c + c1*0.5
a2 = fa(N, ak, bk, beta)*hs
b2 = fb(N, ak, bk, beta, gamma)*hs
c2 = fc(N, bk, gamma)*hs
ak = a + a2*0.5
bk = b + b2*0.5
ck = c + c2*0.5
a3 = fa(N, ak, bk, beta)*hs
b3 = fb(N, ak, bk, beta, gamma)*hs
c3 = fc(N, bk, gamma)*hs
ak = a + a3
bk = b + b3
ck = c + c3
a4 = fa(N, ak, bk, beta)*hs
b4 = fb(N, ak, bk, beta, gamma)*hs
c4 = fc(N, bk, gamma)*hs
a = a +(a1 + 2*(a2 + a3)+ a4)/6
b = b +(b1 + 2*(b2 + b3)+ b4)/6
c = c +(c1 + 2*(c2 + c3)+ c4)/6
return a, b, c
dfBlend=.5*dflgbm +.5*dfnn
dfBlend.loc[:,'object_id']=dflgbm.loc[:,'object_id']<define_search_space>
def SIR(N, b0, beta, gamma, hs):

 a = float(N-1)/N -b0
b = float(1)/N +b0
c = 0.
 sus, inf, rec= [],[],[]
for i in range(10000):
sus.append(a)
 inf.append(b)
 rec.append(c)
 a,b,c = rK4(N, a, b, c, fa, fb, fc, beta, gamma, hs)
 return sus, inf, rec
dfs=[dflgbm, dfnn]
wts=[.5,.5]
npc=1000
def validateBlend(bdf, dfs, wts, npc=1000):
recs=bdf.shape[0]
cols=len(bdf.columns)
 failures=0
for i in range(npc):
record=random.randint(0,recs-1)
 feat=bdf.columns[random.randint(0, cols-1)]
shouldBe=0
isVal=bdf.loc[record, feat]
for j in range(len(wts)) :
shouldBe += dfs[j].loc[record, feat] * wts[j]
shouldBe=round(shouldBe,6)
 isVal=round(isVal,6)
 if shouldBe==isVal:
dosomething=False
else:
print(record)
 print(feat)
 print('should be: ' + str(shouldBe))
print('is: ' + str(isVal))
failures+=1
print('total failures: ' + str(failures)+ ' out of ' + str(npc)+' sample points')
 return failures
failures=validateBlend(dfBlend, dfs, wts)
<save_to_csv>
sus, inf, rec = SIR(N, b0, beta, gamma, hs )
dfBlend.to_csv('lgbm50DecisiveNeural50.csv', index=False )<set_options>
data= pd.read_csv(".. /input/coviddata/time-series-19-covid-combined.csv")
 confirmed_total_date = data.groupby(['Date'] ).agg({'Confirmed':['sum']})
 fatalities_total_date = data.groupby(['Date'] ).agg({'Deaths':['sum']})
 total_date = confirmed_total_date.join(fatalities_total_date)
 confirmed_india = data[data['Country/Region']=='India'].groupby(['Date'] ).agg({'Confirmed':['sum']})
 fatalities_india = data[data['Country/Region']=='India'].groupby(['Date'] ).agg({'Deaths':['sum']})
 total_date_india = confirmed_india.join(fatalities_india)
 pop_india=1380004385
population = float(pop_india)
 country_df = total_date_india[9:]
country_df['day_count'] = list(range(1,len(country_df)+1))
ydata = [i for i in country_df.Confirmed['sum'].values]
xdata = country_df.day_count
ydata = np.array(ydata, dtype=float)
 xdata = np.array(xdata, dtype=float )
np.warnings.filterwarnings('ignore')
<categorify>
N = population
inf0 = ydata[0]
sus0 = N - inf0
rec0 = 0.0
def sir_model(y, x, beta, gamma):
sus = -beta * y[0] * y[1] / N
rec = gamma * y[1]
inf = -(sus + rec)
 return sus, inf, rec
def fit_odeint(x, beta, gamma):
return integrate.odeint(sir_model,(sus0, inf0, rec0), x, args=(beta, gamma)) [:,1]
popt, pcov = optimize.curve_fit(fit_odeint, xdata, ydata)
 fitted = fit_odeint(xdata, *popt)
 plt.plot(xdata, ydata, 'o')
 plt.plot(xdata, fitted)
 plt.title("Fit of SIR model to global infected cases")
 plt.ylabel("Population infected in India")
 plt.xlabel("Days")
 plt.show()
print("Optimal parameters: beta =", popt[0], " and gamma = ", popt[1] )
def multi_weighted_logloss(y_true, y_preds, classes, class_weights):

 y_p = y_preds.reshape(y_true.shape[0], len(classes), order='F')
 y_ohe = pd.get_dummies(y_true)
 y_p = np.clip(a=y_p, a_min=1e-15, a_max=1 - 1e-15)
 y_p_log = np.log(y_p)
 y_log_ones = np.sum(y_ohe.values * y_p_log, axis=0)
 nb_pos = y_ohe.sum(axis=0 ).values.astype(float)
 class_arr = np.array([class_weights[k] for k in sorted(class_weights.keys())])
 y_w = y_log_ones * class_arr / nb_pos
loss = - np.sum(y_w)/ np.sum(class_arr)
 return loss<compute_test_metric>
from scipy.optimize import curve_fit
from statsmodels.tsa.statespace.sarimax import SARIMAX
from statsmodels.tsa.arima_model import ARIMA
from random import random
def lgbm_multi_weighted_logloss(y_true, y_preds):

 classes = [6, 15, 16, 42, 52, 53, 62, 64, 65, 67, 88, 90, 92, 95]
class_weights = {6: 1, 15: 2, 16: 1, 42: 1, 52: 1, 53: 1, 62: 1, 64: 2, 65: 1, 67: 1, 88: 1, 90: 1, 92: 1, 95: 1}
loss = multi_weighted_logloss(y_true, y_preds, classes, class_weights)
 return 'wloss', loss, False
def xgb_multi_weighted_logloss(y_predicted, y_true, classes, class_weights):
loss = multi_weighted_logloss(y_true.get_label() , y_predicted,
classes, class_weights)
 return 'wloss', loss<compute_train_metric>
PATH_WEEK2='/kaggle/input/covid19-global-forecasting-week-2'
df_train = pd.read_csv(f'{PATH_WEEK2}/train.csv')
 df_test = pd.read_csv(f'{PATH_WEEK2}/test.csv')
 df_train.head()
df_test.head()
df_train.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_test.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_train.rename(columns={'Province_State':'State'}, inplace=True)
 df_test.rename(columns={'Province_State':'State'}, inplace=True)
 df_train['Date'] = pd.to_datetime(df_train['Date'], infer_datetime_format=True)
 df_test['Date'] = pd.to_datetime(df_test['Date'], infer_datetime_format=True)
 df_train.info()
df_test.info()
y1_Train = df_train.iloc[:, -2]
y1_Train.head()
y2_Train = df_train.iloc[:, -1]
y2_Train.head()
EMPTY_VAL = "EMPTY_VAL"
def fillState(state, country):
if state == EMPTY_VAL: return country
return state
def save_importances(importances_):
mean_gain = importances_[['gain', 'feature']].groupby('feature' ).mean()
importances_['mean_gain'] = importances_['feature'].map(mean_gain['gain'])
 return importances_<train_model>
X_Train = df_train.copy()
X_Train['State'].fillna(EMPTY_VAL, inplace=True)
 X_Train['State'] = X_Train.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_Train.loc[:, 'Date'] = X_Train.Date.dt.strftime("%m%d")
 X_Train["Date"] = X_Train["Date"].astype(int)
 X_Train.head()
X_Test = df_test.copy()
X_Test['State'].fillna(EMPTY_VAL, inplace=True)
 X_Test['State'] = X_Test.loc[:, ['State', 'Country']].apply(lambda x : fillState(x['State'], x['Country']), axis=1)
 X_Test.loc[:, 'Date'] = X_Test.Date.dt.strftime("%m%d")
 X_Test["Date"] = X_Test["Date"].astype(int)
 X_Test.head()
def smoteAdataset(Xig_train, yig_train, Xig_test, yig_test):
sm=SMOTE(random_state=2)
 Xig_train_res, yig_train_res = sm.fit_sample(Xig_train, yig_train.ravel())
 return Xig_train_res, pd.Series(yig_train_res), Xig_test, pd.Series(yig_test )<compute_train_metric>
le = preprocessing.LabelEncoder()
X_Train.Country = le.fit_transform(X_Train.Country)
 X_Train['State'] = le.fit_transform(X_Train['State'])
 X_Train.head()
X_Test.Country = le.fit_transform(X_Test.Country)
 X_Test['State'] = le.fit_transform(X_Test['State'])
 X_Test.head()
df_train.head()
df_train.loc[df_train.Country == 'Afghanistan', :]
df_test.tail()
def lgbm_modeling_cross_validation(params,
full_train,
y,
classes,
class_weights,
nr_fold=12,
random_state=1):
w = y.value_counts()
weights = {i : np.sum(w)/ w[i] for i in w.index}
clfs = []
importances = pd.DataFrame()
folds = StratifiedKFold(n_splits=nr_fold,
shuffle=True,
random_state=random_state)
 oof_preds = np.zeros(( len(full_train), np.unique(y ).shape[0]))
for fold_,(trn_, val_)in enumerate(folds.split(y, y)) :
trn_x, trn_y = full_train.iloc[trn_], y.iloc[trn_]
val_x, val_y = full_train.iloc[val_], y.iloc[val_]
trn_xa, trn_y, val_xa, val_y=smoteAdataset(trn_x.values, trn_y.values, val_x.values, val_y.values)
 trn_x=pd.DataFrame(data=trn_xa, columns=trn_x.columns)
 val_x=pd.DataFrame(data=val_xa, columns=val_x.columns)
 clf = LGBMClassifier(**params)
 clf.fit(
 trn_x, trn_y,
eval_set=[(trn_x, trn_y),(val_x, val_y)],
eval_metric=lgbm_multi_weighted_logloss,
verbose=100,
early_stopping_rounds=50,
sample_weight=trn_y.map(weights)
)
 clfs.append(clf)
 oof_preds[val_, :] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)
 print('no {}-fold loss: {}'.format(fold_ + 1,
multi_weighted_logloss(val_y, oof_preds[val_, :],
classes, class_weights)))
 imp_df = pd.DataFrame({
'feature': full_train.columns,
'gain': clf.feature_importances_,
'fold': [fold_ + 1] * len(full_train.columns),
})
 importances = pd.concat([importances, imp_df], axis=0, sort=False)
 score = multi_weighted_logloss(y_true=y, y_preds=oof_preds,
classes=classes, class_weights=class_weights)
 print('MULTI WEIGHTED LOG LOSS: {:.5f}'.format(score))
df_importances = save_importances(importances_=importances)
 df_importances.to_csv('lgbm_importances.csv', index=False)
 return clfs, score<predict_on_test>
filterwarnings('ignore')
 le = preprocessing.LabelEncoder()
countries = X_Train.Country.unique()
df_out = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = X_Train.loc[X_Train.Country == country, :].State.unique()
for state in states:
X_Train_CS = X_Train.loc[(X_Train.Country == country)&(X_Train.State == state), ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities']]
y1_Train_CS = X_Train_CS.loc[:, 'ConfirmedCases']
y2_Train_CS = X_Train_CS.loc[:, 'Fatalities']
X_Train_CS = X_Train_CS.loc[:, ['State', 'Country', 'Date']]
X_Train_CS.Country = le.fit_transform(X_Train_CS.Country)
 X_Train_CS['State'] = le.fit_transform(X_Train_CS['State'])
 X_Test_CS = X_Test.loc[(X_Test.Country == country)&(X_Test.State == state), ['State', 'Country', 'Date', 'ForecastId']]
X_Test_CS_Id = X_Test_CS.loc[:, 'ForecastId']
X_Test_CS = X_Test_CS.loc[:, ['State', 'Country', 'Date']]
X_Test_CS.Country = le.fit_transform(X_Test_CS.Country)
 X_Test_CS['State'] = le.fit_transform(X_Test_CS['State'])
 model1 = XGBRegressor(n_estimators=1000)
 model1.fit(X_Train_CS, y1_Train_CS)
 y1_pred = model1.predict(X_Test_CS)
 model2 = XGBRegressor(n_estimators=1000)
 model2.fit(X_Train_CS, y2_Train_CS)
 y2_pred = model2.predict(X_Test_CS)
 df = pd.DataFrame({'ForecastId': X_Test_CS_Id, 'ConfirmedCases': y1_pred, 'Fatalities': y2_pred})
 df_out = pd.concat([df_out, df], axis=0)
def predict_chunk(df_, clfs_, features, train_mean):
agg_ = df_
full_test = agg_.reset_index()
full_test = full_test.fillna(0)
 preds_ = None
for clf in clfs_:
if preds_ is None:
preds_ = clf.predict_proba(full_test[features])/ len(clfs_)
 else:
preds_ += clf.predict_proba(full_test[features])/ len(clfs_)
 preds_99 = np.ones(preds_.shape[0])
 for i in range(preds_.shape[1]):
preds_99 *=(1 - preds_[:, i])
 preds_df_ = pd.DataFrame(preds_, columns=['class_' + str(s)for s in clfs_[0].classes_])
 preds_df_['object_id'] = full_test['object_id']
preds_df_['class_99'] = 0.14 * preds_99 / np.mean(preds_99)
 return preds_df_<save_to_csv>
df_out.ForecastId = df_out.ForecastId.astype('int')
 df_out.tail()
df_out.to_csv('submission.csv', index=False )