Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<train_model>
import numpy as np
import pandas as pd
import math
print('Train start')
 params = {"objective" : "multiclass",
"num_class" : 4,
"num_leaves" : 60,
"max_depth": -1,
"learning_rate" : 0.01,
"bagging_fraction" : 0.9,
"feature_fraction" : 0.9,
"bagging_freq" : 5,
"bagging_seed" : 42,
"verbosity" : -1 }<define_variables>
%%time
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/train.csv' )
y_tra = y_train - 1
y_tes = y_test -1<train_model>
junk =['Id','Date','Province_State']
train.drop(junk, axis=1, inplace=True )
lgtrain, lgval = lgb.Dataset(data = X_train, label = y_tra), lgb.Dataset(data = X_test, label = y_tes)
 print(lgtrain)
 lgb_ad_model = lgb.train(params, lgtrain, num_boost_round = 2000, valid_sets=[lgtrain, lgval],
early_stopping_rounds=100, verbose_eval=100 )<predict_on_test>
train['Country_Region'] = c
predictions = lgb_ad_model.predict(test_set )<define_variables>
train['ConfirmedCases'] = train['ConfirmedCases'].astype(int)
 train['Fatalities'] = train['Fatalities'].astype(int )
predict_df_1 = []
predict_df_2 = []
predict_df_3 = []
predict_df_4 = []<predict_on_test>
def prep_data(train):
X_train = train[train.ConfirmedCases >0]
X_train.reset_index(inplace = True, drop = True)
 train.reset_index(inplace = True, drop = True)
 return(X_train, train )
for i in range(len(predictions)) :
predict_df_1.append(predictions[i][0])
 predict_df_2.append(predictions[i][1])
 predict_df_3.append(predictions[i][2])
 predict_df_4.append(predictions[i][3] )<create_dataframe>
def Calculate_Table(X_train):
diff_conf, conf_old = [], 0
diff_fat, fat_old = [], 0
dd_conf, dc_old = [], 0
dd_fat, df_old = [], 0
ratios = []
for row in X_train.values:
diff_conf.append(row[1]-conf_old)
 conf_old = row[1]
diff_fat.append(row[2]-fat_old)
 fat_old = row[2]
dd_conf.append(diff_conf[-1]-dc_old)
 dc_old = diff_conf[-1]
dd_fat.append(diff_fat[-1]-df_old)
 df_old = diff_fat[-1]
ratios.append(fat_old / conf_old)
 ratio = fat_old / conf_old
return diff_conf, conf_old, diff_fat, fat_old, dd_conf, dc_old, dd_fat, df_old, ratios, ratio
df_pred = pd.DataFrame(predict_df_1, columns = ['1'] )<prepare_output>
def populate_df_features(X_train,diff_conf, diff_fat, dd_conf, dd_fat, ratios):
pd.options.mode.chained_assignment = None
X_train['diff_confirmed'] = diff_conf
X_train['diff_fatalities'] = diff_fat
X_train['dd_confirmed'] = dd_conf
X_train['dd_fatalities'] = dd_fat
X_train['ratios'] = ratios
return X_train
df_pred['2'] = predict_df_2
df_pred['3'] = predict_df_3
df_pred['4'] = predict_df_4
df_pred.head()<feature_engineering>
def fill_nan(variable):
if math.isnan(variable):
return 0
else:
return variable
df_pred['Target'] = df_pred[['1', '2', '3', '4']].idxmax(axis = 1 )<save_to_csv>
def Cal_Series_Avg(X_train,ratio):
d_c = fill_nan(X_train.diff_confirmed[X_train.diff_confirmed != 0].mean())
 dd_c = fill_nan(X_train.dd_confirmed[X_train.dd_confirmed != 0].mean())
 d_f = fill_nan(X_train.diff_fatalities[X_train.diff_fatalities != 0].mean())
 dd_f = fill_nan(X_train.dd_fatalities[X_train.dd_fatalities != 0].mean())
 rate = fill_nan(X_train.ratios[X_train.ratios != 0].mean())
 rate = max(rate,ratio)
 return d_c, dd_c, d_f, dd_f, rate
df_pred.to_csv('result.csv', index = False )<set_options>
def apply_taylor(train, d_c, dd_c, d_f, dd_f, rate):
pred_c, pred_f = list(train.ConfirmedCases.loc[57:69].astype(int)) , list(train.Fatalities.loc[57:69].astype(int))
for i in range(1, 31):
pred_c.append(int(( train.ConfirmedCases[69] + d_c*i + 0.5*dd_c*(i**2))))
pred_f.append(pred_c[-1]*rate)
 return pred_c, pred_f
%matplotlib inline 
<load_from_csv>
def apply_taylor2(train, d_c, dd_c, d_f, dd_f, rate):
pred_c, pred_f = list(train.ConfirmedCases.loc[57:58].astype(int)) , list(train.Fatalities.loc[57:58].astype(int))
for i in range(1, 42):
pred_c.append(int(( train.ConfirmedCases[58] + d_c*i + 0.5*dd_c*(i**2))))
pred_f.append(pred_c[-1]*rate)
 return pred_c, pred_f
data=pd.read_csv(".. /input/train.csv")
 data1=pd.read_csv(".. /input/test.csv" )<feature_engineering>
pc = []
pf = []
pc2 = []
pf2 = []
pred_c = []
pred_f = []
pred_c2 = []
pred_f2 = []
for i,country in enumerate(country_list):
country_data = train[train['Country_Region'] == country]
X_train, country_data = prep_data(country_data)
 if(len(X_train)> 0):
diff_conf, conf_old, diff_fat, fat_old, dd_conf, dc_old, dd_fat, df_old, ratios, ratio = Calculate_Table(X_train)
 X_train = populate_df_features(X_train,diff_conf, diff_fat, dd_conf, dd_fat, ratios)
 d_c, dd_c, d_f, dd_f, rate = Cal_Series_Avg(X_train, ratio)
 pred_c, pred_f = apply_taylor(country_data, d_c, dd_c, d_f, dd_f, rate)
 pred_c2, pred_f2 = apply_taylor2(country_data, d_c, dd_c, d_f, dd_f, rate)
 else:
pred_c = list(np.zeros(43))
pred_f = list(np.zeros(43))
pred_c2 = list(np.zeros(43))
pred_f2 = list(np.zeros(43))
pc += pred_c
pf += pred_f
pc2 += pred_c2
pf2 += pred_f2
df = data[['epared1','epared2','epared3']]
x = df.stack()
data['epared'] = np.array(pd.Categorical(x[x!=0].index.get_level_values(1)))
 data['epared'] = data['epared'].apply(lambda x : 1 if x == 'epared1' else(2 if x == 'epared2' else 3))
df = data[['etecho1','etecho2','etecho3']]
x = df.stack()
data['etecho'] = np.array(pd.Categorical(x[x!=0].index.get_level_values(1)))
 data['etecho'] = data['epared'].apply(lambda x : 1 if x == 'etecho1' else(2 if x == 'etecho2' else 3))
df = data[['eviv1','eviv2','eviv3']]
x = df.stack()
data['eviv'] = np.array(pd.Categorical(x[x!=0].index.get_level_values(1)))
 data['eviv'] = data['epared'].apply(lambda x : 1 if x == 'eviv1' else(2 if x == 'eviv2' else 3))
data.drop(['epared1','epared2','epared3','etecho1','etecho2','etecho3','eviv1','eviv2','eviv3'],axis=1,inplace=True )<feature_engineering>
pc = list(map(int, pc))
pf = list(map(int, pf))
pc2 = list(map(int, pc2))
pf2 = list(map(int, pf2))
df = data1[['epared1','epared2','epared3']]
x = df.stack()
data1['epared'] = np.array(pd.Categorical(x[x!=0].index.get_level_values(1)))
 data1['epared'] = data1['epared'].apply(lambda x : 1 if x == 'epared1' else(2 if x == 'epared2' else 3))
df = data1[['etecho1','etecho2','etecho3']]
x = df.stack()
data1['etecho'] = np.array(pd.Categorical(x[x!=0].index.get_level_values(1)))
 data1['etecho'] = data1['epared'].apply(lambda x : 1 if x == 'etecho1' else(2 if x == 'etecho2' else 3))
df = data1[['eviv1','eviv2','eviv3']]
x = df.stack()
data1['eviv'] = np.array(pd.Categorical(x[x!=0].index.get_level_values(1)))
 data1['eviv'] = data1['epared'].apply(lambda x : 1 if x == 'eviv1' else(2 if x == 'eviv2' else 3))
data1.drop(['epared1','epared2','epared3','etecho1','etecho2','etecho3','eviv1','eviv2','eviv3'],axis=1,inplace=True )<categorify>
import matplotlib.pyplot as plt
def data_cleaning(data):
data['dependency']=np.sqrt(data['SQBdependency'])
 data['rez_esc']=data['rez_esc'].fillna(0)
 data['v18q1']=data['v18q1'].fillna(0)
 data['v2a1']=data['v2a1'].fillna(0)
 conditions = [ 
(data['edjefe']=='no')&(data['edjefa']=='no'), 
(data['edjefe']=='yes')&(data['edjefa']=='no'), 
(data['edjefe']=='no')&(data['edjefa']=='yes'), 
(data['edjefe']!='no')&(data['edjefe']!='yes')&(data['edjefa']=='no'), 
(data['edjefe']=='no')&(data['edjefa']!='no')
 ]
choices = [0, 1, 1, data['edjefe'], data['edjefa']]
data['edjefx']=np.select(conditions, choices)
 data['edjefx']=data['edjefx'].astype(int)
 data.drop(['edjefe', 'edjefa'], axis=1, inplace=True)
 meaneduc_nan=data[data['meaneduc'].isnull() ][['Id','idhogar','escolari']]
me=meaneduc_nan.groupby('idhogar')['escolari'].mean().reset_index()
for row in meaneduc_nan.iterrows() :
idx=row[0]
idhogar=row[1]['idhogar']
m=me[me['idhogar']==idhogar]['escolari'].tolist() [0]
data.at[idx, 'meaneduc']=m
data.at[idx, 'SQBmeaned']=m*m
return data 
<sort_values>
my_submission = pd.DataFrame({'ForecastId': list(range(1,len(pc2)+1)) , 'ConfirmedCases': pc2, 'Fatalities': pf2})
 print(my_submission)
 my_submission.to_csv('submission.csv', index=False )