Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<data_type_conversions>
datapath = '.. /input/covid19-global-forecasting-week-4/'
datapath2 = '.. /input/worldpopulationinfo/'
datapath3 = '.. /input/country-ppp/'
datapath4 = '.. /input/populationandcountryinfo/'
datapath5 = '.. /input/usstateland/'
datapath_week1 = '.. /input/covid19week1/'
add_other = True
CURVE_SMOOTHING = True
USE_NEW = False
normfactor = 1.0
NUM_SHIFT = 25
days_shift =[0,1,2,3,5,7,10,14,18,22,27,32,37,42]
days_shift =[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,16,18,20,22,24,26,28,30,32,34,36]
NUM_MODELS = 10
TRAIN_START_DAY = 1
TARGETS = ["ConfirmedCases", "Fatalities"]
if USE_NEW:
TARGETS = ["NewCases", "NewFatalities"]
loc_group = ["Province_State", "Country_Region"]
if add_other:
base_features = ['ages 0-14', 'ages 15-64', 'ages 64-',
'Population(2020)', 'Density(P/Km²)', 'Med.Age', 'Urban Pop %',
'Apr', 'year', 'Low Temp', 'ppp', 'Season',
'R_Africa','R_China',
'R_Australia and New Zealand', 'R_Eastern Asia', 'R_Europe',
'R_Latin America and the Caribbean', 'R_Northern America', 'R_Oceania',
'R_Southern Asia', 'R_Western Asia',
'I_High income', 'I_Low income', 'I_Lower middle income',
'I_Upper middle income']
base_features = ['ages 64-',
'Urban Pop %',
'Season','Apr','ppp']
if USE_NEW:
base_features =['NewFatalities_ppm','NewCases_ppm'] + base_features
else:
a=1
else:
base_features=[]
def get_shift_features() :
shift_features = []
for s in range(1, NUM_SHIFT+1):
for col in TARGETS:
shift_features.append("prev_{}_{}".format(col, days_shift[s]))
return shift_features
shift_features = get_shift_features()
prev_targets = shift_features[0:2]
def fill_shift_columns(df,targets):
for s in range(1, NUM_SHIFT+1):
for col in targets:
df["prev_{}_{}".format(col, days_shift[s])] = df.groupby(loc_group)[col].shift(days_shift[s])
 return df
def add_seasons(coor_df):
coor_df["Season"] = 0
mask = coor_df["Lat"]>20
coor_df.loc[mask,"Season"] = 1
mask = coor_df["Lat"]<-20
coor_df.loc[mask,"Season"] = -1
return coor_df
def preprocess(df):
df["Date"] = df["Date"].astype("datetime64[ms]")
 df["days"] =(df["Date"] - pd.to_datetime("2020-01-01")).dt.days
for col in loc_group:
df[col].fillna("none", inplace=True)
 return df
def print_rows_with_nan(df,features=False):
if not features:
dyt = df.isna()
dyt2 = dyt.any(axis=1)
 dyt2.sum()
print(df.loc[dyt2])
 else:
dyt = df[features].isna()
dyt2 = dyt.any(axis=1)
 dyt2.sum()
print(df[features].loc[dyt2])
 return
def diff(first, second):
second = set(second)
 return [item for item in first if item not in second]
def has_nan(df,features=False):
if not features:
dyt = df.isnull()
else:
dyt = df[features].isnull()
dyt2 = dyt.any(axis=1)
 return dyt2.sum()
def print_washinton(df,features,values):
print(len(values))
if len(values)== 1:
mask1 = df[features[0]]==values[0]
print(df.loc[mask1,features+["Date","ConfirmedCases","Fatalities"]].tail(50))
elif len(values)== 2:
mask1 = df[features[0]]==values[0]
mask2 = df[features[1]]==values[1]
mask4 = pd.concat(( mask1,mask2), axis=1)
 mask5 = mask4.all(axis=1)
 print(df.loc[mask5,features+["Date","ConfirmedCases","Fatalities"]].tail(50))
else:
mask1 = df[features[0]]==values[0]
mask2 = df[features[1]]==values[1]
mask3 = df[features[2]]==values[2]
mask4 = pd.concat(( mask1,mask2,mask3), axis=1)
 mask5 = mask4.all(axis=1)
 print(df.loc[mask5,["Date"]])
 return
def find_anomalities(df):
df[['Cases_test']] = df.groupby([ 'Country_Region', 'Province_State'])\
[['ConfirmedCases']].shift(1)
 df[['Fatality_test']] = df.groupby(['Country_Region', 'Province_State'])\
[['Fatalities']].shift(1)
 mask1 = df['ConfirmedCases'] < df['Cases_test']
mask2 = df['Fatalities'] < df['Fatality_test']
print("ANOMALITIES ConfirmedCases")
 print(df.loc[mask1])
 print("ANOMALITIES Fatalities")
 print(df.loc[mask2])
 print("SLUT")
 return
def correct_anomalities(df):
df[['Cases_testm']] = df.groupby([ 'Country_Region', 'Province_State'])\
[['ConfirmedCases']].shift(1)
 df[['Fatalities_testm']] = df.groupby(['Country_Region', 'Province_State'])\
[['Fatalities']].shift(1)
 df[['Cases_testp']] = df.groupby([ 'Country_Region', 'Province_State'])\
[['ConfirmedCases']].shift(-1)
 df[['Fatalities_testp']] = df.groupby(['Country_Region', 'Province_State'])\
[['Fatalities']].shift(-1)
 print("CORRECTING ConfirmedCases")
 mask1 = df['ConfirmedCases'] < df['Cases_testm']
mask2 = df['Cases_testp'] >= df['Cases_testm']
mask3 = pd.concat(( mask1,mask2),axis=1)
 mask4 = mask3.all(axis=1)
 df.loc[mask4,'ConfirmedCases'] = 0.5*(df.loc[mask4,'Cases_testp']+df.loc[mask4,'Cases_testm'])
 print(df.loc[mask4,['Country_Region', 'Province_State','Cases_testm','ConfirmedCases','Cases_testp']])
 mask1 = df['ConfirmedCases'] > df['Cases_testm']
mask2 = df['Cases_testp'] == df['Cases_testm']
mask3 = pd.concat(( mask1,mask2),axis=1)
 mask4a = mask3.all(axis=1)
 df.loc[mask4a,'ConfirmedCases'] = 0.5*(df.loc[mask4a,'Cases_testp']+df.loc[mask4a,'Cases_testm'])
 print(df.loc[mask4a,['Country_Region', 'Province_State','Cases_testm','ConfirmedCases','Cases_testp']])
 print("CORRECTED ConfirmedCases:", mask4.sum() ,"+", mask4a.sum())
 print("CORRECTING Fatalities")
 mask1 = df['Fatalities'] < df['Fatalities_testm']
mask2 = df['Fatalities_testp'] >= df['Fatalities_testm']
mask3 = pd.concat(( mask1,mask2),axis=1)
 mask4 = mask3.all(axis=1)
 df.loc[mask4,'Fatalities'] = 0.5*(df.loc[mask4,'Fatalities_testp']+df.loc[mask4,'Fatalities_testm'])
 print(df.loc[mask4,['Country_Region', 'Province_State','Fatalities_testm','Fatalities','Fatalities_testp']])
 mask1 = df['Fatalities'] > df['Fatalities_testm']
mask2 = df['Fatalities_testp'] == df['Fatalities_testm']
mask3 = pd.concat(( mask1,mask2),axis=1)
 mask4a = mask3.all(axis=1)
 df.loc[mask4a,'Fatalities'] = 0.5*(df.loc[mask4a,'Fatalities_testp']+df.loc[mask4a,'Fatalities_testm'])
 print(df.loc[mask4a,['Country_Region', 'Province_State','Fatalities_testm','Fatalities','Fatalities_testp']])
 print("CORRECTED Fatalities:",mask4.sum() ,"+",mask4a.sum())
 print("CORRECTING Series(in Mix)")
 cases = 1
sumcases = 0
while cases > 0:
mask1 = df['ConfirmedCases'] < df['Cases_testm']
df.loc[mask1,'ConfirmedCases'] = df.loc[mask1,'Cases_testm']
print("Cases",df.loc[mask1,['Country_Region', 'Province_State','Cases_testm','ConfirmedCases']])
 mask2 = df['Fatalities'] < df['Fatalities_testm']
df.loc[mask2,'Fatalities'] = df.loc[mask2,'Fatalities_testm']
print("Fatas",df.loc[mask2,['Country_Region', 'Province_State','Fatalities_testm','Fatalities']])
 cases = mask1.sum() +mask2.sum()
sumcases+=cases
print
df[['Cases_testm']] = df.groupby([ 'Country_Region', 'Province_State'])\
[['ConfirmedCases']].shift(1)
 df[['Fatalities_testm']] = df.groupby(['Country_Region', 'Province_State'])\
[['Fatalities']].shift(1)
 df.drop([ 'Cases_testm', 'Fatalities_testm', 'Cases_testp','Fatalities_testp'],axis=1,inplace=True)
 print("Corrected in Series in all",sumcases)
 return
part_agg = questions_df.groupby('part')['content_uncorrect_count'].agg(['sum'])
 questions_df['part_uncor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32')
 part_agg = questions_df.groupby('part')['content_correct_count'].agg(['sum'])
 questions_df['part_cor_count'] = questions_df['part'].map(part_agg['sum'] ).astype('int32' )<categorify>
population_by_age_df = pd.read_csv(datapath4 + "population_age_info.csv")
 population_by_age_df.drop('ID',axis=1,inplace=True)
 population_by_age_df[['ages 0-14', 'ages 15-64','Density(P/Km²)','Med.Age', 'ages 64-','Urban Pop %']] = \
population_by_age_df[['ages 0-14', 'ages 15-64','Density(P/Km²)','Med.Age', 'ages 64-','Urban Pop %']].apply(lambda x: x.fillna(x.mean()))
translations = [["Brunei Darussalam","Brunei"],
["Myanmar","Bruma"],
["Gambia, The","Gambia"],
["Egypt, Arab Rep.","Egypt"],
["Congo, Rep.","Congo(Brazzaville)"],
["Congo, Dem.Rep.","Congo(Kinshasa)"],
["Iran, Islamic Rep.","Iran"],
["Korea, Rep.","Korea, South"],
["Russian Federation","Russia"],
["Syrian Arab Republic","Syria"],
["Venezuela, RB","Venezuela"],
["Slovak Republic","Slovakia"]]
for post in translations:
mask = population_by_age_df["Country_Region"] == post[0]
population_by_age_df.loc[mask,"Country_Region"] = post[1]
usstates_land = pd.read_csv(datapath5 + "us-state-land.csv", sep='\t')
 usstates_population = pd.read_csv(datapath2 + "us-state-population.csv", sep='\t')
 usstates_info = pd.merge(usstates_land,usstates_population, on=['Province_State'], how='left')
 usstates_info['Country_Region'] = 'US'
usstates_info["Province_State"].replace('_',' ', regex=True,inplace=True)
 supp_info = pd.read_csv(datapath2 + "population_info_supplement.csv")
 supp_info["Country_Region"].replace(':',',', regex=True,inplace=True)
 CountryRegion = pd.read_csv(datapath2+'CountryRegion.csv')
 CountryRegion.drop('ID',axis=1,inplace=True)
 state_temperatures = pd.read_csv(datapath4+'state_temperature_info.csv')
 state_temperatures.drop('ID',axis=1,inplace=True)
 state_temperatures["Province_State"].fillna("", inplace=True)
 ppp_tabel = pd.read_csv(datapath3 + 'Country_PPP.csv', sep='\s+')
 ppp_tabel.drop('Id', 1,inplace=True)
 ppp_tabel = ppp_tabel.append({'Country' : 'Burma' , 'ppp' : 8000} , ignore_index=True)
 ppp_tabel = ppp_tabel.append({'Country' : 'MS_Zaandam' , 'ppp' : 40000} , ignore_index=True)
 ppp_tabel = ppp_tabel.append({'Country' : 'West_Bank_and_Gaza' , 'ppp' : 20000} , ignore_index=True)
 ppp_tabel["Country"].replace('_',' ', regex=True,inplace=True)
 ppp_tabel["Country"].replace('United States','US', regex=True,inplace=True)
 ppp_tabel.rename(columns={'Country':'Country_Region'},inplace=True)
 ppp_tabel.sort_values('Country_Region',inplace=True)
 coor_df = pd.read_csv(datapath_week1 + "train.csv" ).rename(columns={"Country/Region": "Country_Region","Province/State":"Province_State"})
 coor_df["Province_State"].fillna("", inplace=True)
 coor_df = coor_df[coor_df["Country_Region"].notnull() ]
coor_df = coor_df.groupby(["Country_Region","Province_State"])[["Lat", "Long"]].mean().reset_index()
coor_df = add_seasons(coor_df)
 coor_country = coor_df.groupby(["Country_Region"])[["Lat", "Long"]].mean().reset_index()
coor_country = add_seasons(coor_country)
 coor_country.rename(columns={'Season':'Season2'},inplace=True)
 coor_df = pd.merge(coor_df,coor_country, on=['Country_Region'], how='left')
bundle_agg = questions_df.groupby('bundle_id')['content_correctness'].agg(['mean'])
 questions_df['bundle_correctness_mean'] = questions_df['bundle_id'].map(bundle_agg['mean'])
 questions_df.bundle_correctness_mean=questions_df.bundle_correctness_mean.astype('float16' )<drop_column>
def add_other_info(df,istest_df,usstates_info,supp_info,population_by_age_df,CountryRegion,state_temperatures,ppp_tabel,coor_df):
df = pd.merge(df, usstates_info, on=['Country_Region','Province_State'], how='left')
 df = pd.merge(df, population_by_age_df, on=['Country_Region'], how='left')
 df[['ages 0-14', 'ages 15-64','Density(P/Km²)','Med.Age', 'ages 64-','Urban Pop %']] = \
df[['ages 0-14', 'ages 15-64','Density(P/Km²)','Med.Age', 'ages 64-','Urban Pop %']].apply(lambda x: x.fillna(x.mean()))
print(df[df['IncomeGroup'].isnull() ]["Country_Region"].unique())
 df = pd.merge(df, CountryRegion, on=['Country_Region'], how='left')
 mask = df['Country_Region']=='US'
df.loc[mask,'Population(2020)'] = df.loc[mask,'population']
df.loc[mask,'Density(P/Km²)'] = df.loc[mask,'population']/df.loc[mask,'land']
df.drop(["population","land"],axis=1,inplace=True)
 df = pd.merge(df, supp_info, on=['Country_Region'], how='left')
 mask = df['Population(2020)'].isnull()
df.loc[mask,'Population(2020)'] = df.loc[mask,'population']
df.loc[mask,'Density(P/Km²)'] = df.loc[mask,'population']/df.loc[mask,'land']
df.drop(["population","land"],axis=1,inplace=True)
 df = pd.merge(df, state_temperatures, on=['Country_Region','Province_State'], how='left')
 df[['Jan', 'Feb', 'Mar', 'Apr', 'May',
'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec', 'year', 'Low Temp']] = \
df[['Jan', 'Feb', 'Mar', 'Apr', 'May',
'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec', 'year', 'Low Temp']].apply(lambda x: x.fillna(x.mean()))
df.drop(['May','Jun', 'Jul', 'Aug', 'Sep', 'Oct','Nov', 'Dec'],axis=1,inplace=True)
 df.drop(['Jan', 'Feb', 'Mar'],axis=1,inplace=True)
 df = pd.merge(df,ppp_tabel, on=['Country_Region'], how='left')
 df = pd.merge(df,coor_df[["Country_Region",'Province_State',"Season","Season2"]],
on=["Country_Region",'Province_State'], how="left")
 mask = df['Season'].isna()
df.loc[mask,["Season"]]= df.loc[mask,"Season2"]
df["Season"].fillna(1, inplace=True)
 mask = df['Country_Region']== 'China'
df.loc[mask,["sub-region"]]= 'China'
onehot = pd.get_dummies(df['sub-region'],prefix='R')
 onehot2 = pd.get_dummies(df['IncomeGroup'],prefix='I')
 df.drop(["Season2","sub-region",'IncomeGroup'],axis=1,inplace= True)
 df = pd.concat([df,onehot],axis=1,ignore_index=False)
 df = pd.concat([df,onehot2],axis=1,ignore_index=False)
 if istest_df:
if USE_NEW:
df["NewCases_ppm"] = np.log1p(1000000/df["Population(2020)"])
 df["NewFatalities_ppm"] = np.log1p(1000000/df["Population(2020)"])
 else:
df["ConfirmedCases_ppm"] = np.log1p(1000000/df["Population(2020)"])
 df["Fatalities_ppm"] = np.log1p(1000000/df["Population(2020)"])
 else:
if USE_NEW:
df["NewCases_ppm"] = np.log1p(df["NewCases"]*1000000/df["Population(2020)"])
 df["NewFatalities_ppm"] = np.log1p(df["NewFatalities"]*1000000/df["Population(2020)"])
 else:
df["ConfirmedCases_ppm"] = np.log1p(df["ConfirmedCases"]*1000000/df["Population(2020)"])
 df["Fatalities_ppm"] = np.log1p(df["Fatalities"]*1000000/df["Population(2020)"])
 df['ages 0-14'] = df['ages 0-14']/100.
 df['ages 15-64'] = df['ages 15-64']/100.
 df['ages 64-'] = df['ages 64-']/100.
 df['Population(2020)'] = np.log1p(df['Population(2020)'])
 df['Density(P/Km²)'] = np.log1p(df['Density(P/Km²)'])
 df['Med.Age'] = df['Med.Age']/50.
 df['Urban Pop %'] = df['Urban Pop %']/100.
 df['Apr'] = df['Apr']/30.
 df['year'] = df['year']/30.
 df['Low Temp'] = df['Low Temp']/30.
 df['ppp'] = np.log1p(df['ppp'])
 return df
del content_agg
del bundle_agg
del part_agg
gc.collect()<define_variables>
df = pd.read_csv(datapath + "train.csv")
 sub_df = pd.read_csv(datapath + "test.csv")
 df['Province_State'].fillna('', inplace=True)
 sub_df['Province_State'].fillna('', inplace=True)
 gem_targets = df[['Country_Region','Province_State','Date']+TARGETS]
gem_targets["Date"] = gem_targets["Date"].astype("datetime64[ms]")
 combi= df.groupby(["Country_Region","Province_State"] ).size().reset_index()
combi =combi[["Country_Region","Province_State"]].values.tolist()
mask1 = df["Country_Region"]=='Guyana'
mask2 = df["Date"] > '2020-03-21'
mask3 = df["Date"] < '2020-03-28'
mask4 = pd.concat(( mask1,mask2,mask3),axis=1)
 mask5 = mask4.all(axis=1)
 df.loc[mask5,"ConfirmedCases"] = 7.0
mask1 = df["Province_State"]=='Northern Territory'
mask2 = df["Date"] > '2020-03-05'
mask3 = df["Date"] < '2020-03-10'
mask4 = pd.concat(( mask1,mask2,mask3),axis=1)
 mask5 = mask4.all(axis=1)
 df.loc[mask5,"ConfirmedCases"] = 1.0
mask1 = df["Country_Region"]=='Philippines'
mask2 = df["Date"] > '2020-03-17'
mask3 = df["Date"] < '2020-03-19'
mask4 = pd.concat(( mask1,mask2,mask3),axis=1)
 mask5 = mask4.all(axis=1)
 df.loc[mask5,"Fatalities"] = 14.0
correct_anomalities(df)
 if USE_NEW:
df[['NewCases','NewFatalities']] = df.groupby(['Country_Region', 'Province_State'])\
[['ConfirmedCases','Fatalities']].transform(lambda x: x.diff())
 mask = df['NewCases'].isnull()
df.loc[mask,['NewCases']] = 0
df.loc[mask,['NewFatalities']] = 0
provins = 'Virgin Islands'
print("testprint for",provins)
 print(df[df['Province_State']== provins][['NewCases','NewFatalities']])
 def row_count(filename):
with open(filename)as in_file:
return sum(1 for _ in in_file)
 df = preprocess(df)
 sub_df = preprocess(sub_df)
 GIVEN_FIRST_DATE = df["Date"].min()
GIVEN_LAST_DATE = df["Date"].max()
SUBMISSION_FIRST_DATE = sub_df["Date"].min()
ESTIMATE_FIRST_DATE = df["Date"].max() + timedelta(days=1)
 ESTIMATE_LAST_DATE = sub_df["Date"].max()
ESTIMATE_DAYS =(ESTIMATE_LAST_DATE - ESTIMATE_FIRST_DATE ).days +1
if add_other:
df = add_other_info(df,False,usstates_info,supp_info,population_by_age_df,CountryRegion,state_temperatures,ppp_tabel,coor_df)
 sub_df = add_other_info(sub_df,True,usstates_info,supp_info,population_by_age_df,CountryRegion,state_temperatures,ppp_tabel,coor_df)
 print("SUB_DF COLUMNS NYLAVET:")
 print(sub_df.columns)
features_dict = {
'timestamp':'float16',
'user_interaction_count':'int16',
'user_interaction_timestamp_mean':'float32',
'lagtime':'float32',
'lagtime2':'float32',
'lagtime3':'float32',
'content_id':'int16',
'task_container_id':'int16',
'user_lecture_sum':'int16',
'user_lecture_lv':'float16',
'prior_question_elapsed_time':'float32',
'delta_prior_question_elapsed_time':'int32',
'user_correctness':'float16',
'user_uncorrect_count':'int16',
'user_correct_count':'int16',
'content_correctness_std':'float16',
'content_correct_count':'int32',
'content_uncorrect_count':'int32',
'content_elapsed_time_mean':'float16',
'content_had_explanation_mean':'float16',
'content_explation_false_mean':'float16',
'content_explation_true_mean':'float16',
'explation_impact':'float16',
'task_container_correctness':'float16',
'task_container_std':'float16',
'task_container_cor_count':'int32',
'task_container_uncor_count':'int32',
'attempt_no':'int8',
'part':'int8',
'part_correctness_mean':'float16',
'part_correctness_std':'float16',
'part_uncor_count':'int32',
'part_cor_count':'int32',
'tags0': 'int8',
'tags1': 'int8',
'tags2': 'int8',
'tags3': 'int8',
'tags4': 'int8',
'tags5': 'int8',
'part_bundle_id':'int32',
'content_sub_bundle':'int8',
'prior_question_had_explanation':'int8',
'explanation_mean':'float16',
'explanation_false_count':'int16',
'explanation_true_count':'int16'
}
categorical_columns= [
'content_id',
'task_container_id',
'part',
'tags0',
'tags1',
'tags2',
'tags3',
'tags4',
'tags5',
'part_bundle_id',
'content_sub_bundle',
'prior_question_had_explanation'
]
features=list(features_dict.keys())
<define_variables>
if CURVE_SMOOTHING:
df['Cases_m'] = df.groupby(['Country_Region', 'Province_State'])[['ConfirmedCases']].transform(lambda x: x.shift(1))
df['Cases_p'] = df.groupby(['Country_Region', 'Province_State'])[['ConfirmedCases']].transform(lambda x: x.shift(-1))
df['Cases_ave'] = 0.5*(df['ConfirmedCases']+0.5*(df['Cases_p']+df['Cases_m']))
case_cols = ['ConfirmedCases','Cases_m','Cases_p','Cases_ave']
df['Fatalities_m'] = df.groupby(['Country_Region', 'Province_State'])[['Fatalities']].transform(lambda x: x.shift(1))
df['Fatalities_p'] = df.groupby(['Country_Region', 'Province_State'])[['Fatalities']].transform(lambda x: x.shift(-1))
df['Fatalities_ave'] = 0.5*(df['Fatalities']+0.5*(df['Fatalities_p']+df['Fatalities_m']))
fata_cols = ['Fatalities','Fatalities_m','Fatalities_p','Fatalities_ave']
if USE_NEW:
df[['NewCases','NewFatalities']] = df.groupby(['Country_Region', 'Province_State'])\
[['ConfirmedCases','Fatalities']].transform(lambda x: x.diff())
 df['NewCases_m'] = df.groupby(['Country_Region', 'Province_State'])[['NewCases']].transform(lambda x: x.shift(1))
df['NewCases_p'] = df.groupby(['Country_Region', 'Province_State'])[['NewCases']].transform(lambda x: x.shift(-1))
df['NewCases_m2'] = df.groupby(['Country_Region', 'Province_State'])[['NewCases']].transform(lambda x: x.shift(2))
df['NewCases_p2'] = df.groupby(['Country_Region', 'Province_State'])[['NewCases']].transform(lambda x: x.shift(-2))
df['NewCases_ave'] = 0.2*(df['NewCases']+df['NewCases_p']+df['NewCases_m']+df['NewCases_p2']+df['NewCases_m2'])
 df['NewFatalities_m'] = df.groupby(['Country_Region', 'Province_State'])[['NewFatalities']].transform(lambda x: x.shift(1))
df['NewFatalities_p'] = df.groupby(['Country_Region', 'Province_State'])[['NewFatalities']].transform(lambda x: x.shift(-1))
df['NewFatalities_m2'] = df.groupby(['Country_Region', 'Province_State'])[['NewFatalities']].transform(lambda x: x.shift(2))
df['NewFatalities_p2'] = df.groupby(['Country_Region', 'Province_State'])[['NewFatalities']].transform(lambda x: x.shift(-2))
df['NewFatalities_ave'] =0.2*(df['NewFatalities']+df['NewFatalities_p']+df['NewFatalities_m']+df['NewFatalities_p2']+df['NewFatalities_m2'])
 date_max = df["Date"].max()
mask = df["Date"] ==date_max
mask2 = df["Date"]==date_max - timedelta(days=1)
 df.loc[mask,'Cases_ave'] = 0.75*df.loc[mask,'ConfirmedCases']+0.25*df.loc[mask,'Cases_m']
df.loc[mask,'Fatalities_ave'] = 0.75*df.loc[mask,'Fatalities'] +0.25*df.loc[mask,'Fatalities_m']
df.loc[mask,'Cases_ave'] = 0.75*df.loc[mask,'ConfirmedCases']+0.25*df.loc[mask,'Cases_m']
df.loc[mask,'Fatalities_ave'] = 0.75*df.loc[mask,'Fatalities'] +0.25*df.loc[mask,'Fatalities_m']
df.drop(['Cases_m', 'Cases_p', 'Fatalities_m','Fatalities_p','ConfirmedCases','Fatalities'],axis=1,inplace=True)
 df.rename(columns={'Cases_ave':'ConfirmedCases','Fatalities_ave':'Fatalities'},inplace=True)
 if USE_NEW:
df.loc[mask2,'NewCases_ave'] = 0.3*df.loc[mask2,'NewCases_p']+0.3*df.loc[mask2,'NewCases']+\
0.3*df.loc[mask2,'NewCases_m']+0.1*df.loc[mask2,'NewCases_m2']
df.loc[mask2,'NewFatalities_ave'] = 0.3*df.loc[mask2,'NewFatalities_p']+0.3*df.loc[mask2,'NewFatalities']+\
0.3*df.loc[mask2,'NewFatalities_m']+0.1*df.loc[mask2,'NewFatalities_m2']
df.loc[mask,'NewCases_ave'] = 0.55*df.loc[mask,'NewCases'] +0.3*df.loc[mask,'NewCases_m']+\
0.2*df.loc[mask,'NewCases_m2']
df.loc[mask,'NewFatalities_ave'] = 0.5*df.loc[mask,'NewFatalities'] +0.3*df.loc[mask,'NewFatalities_m']+\
0.2*df.loc[mask,'NewFatalities_m2']
df.drop(['NewCases_m', 'NewCases_p', 'NewFatalities_m','NewFatalities_p','NewCases','NewFatalities'],axis=1,inplace=True)
 df.drop(['NewCases_m2', 'NewCases_p2', 'NewFatalities_m2','NewFatalities_p2'],axis=1,inplace=True)
 df.rename(columns={'NewCases_ave':'NewCases','NewFatalities_ave':'NewFatalities'},inplace=True)
 if add_other:
df["NewCases_ppm"] = np.log1p(df["NewCases"]*1000000/df["Population(2020)"])
 df["NewFatalities_ppm"] = np.log1p(df["NewFatalities"]*1000000/df["Population(2020)"])
 df.fillna(0, inplace=True)
 sub_df.fillna(0,inplace =True)
 DEFAULT_VALUE = 0
if df is None:
df = DEFAULT_VALUE
if sub_df is None:
sub_df = DEFAULT_VALUE
if has_nan(df,base_features):
print("DYT-DYYYYYT: der er NAN")
 print_rows_with_nan(df)
 else:
print("HURRA, HURRA")
 if has_nan(df)>0:
print("DOOOOOOOOOT: der er NAN")
 print_rows_with_nan(df)
 print("DOT DONE")
 else:
print("HIP HIP")
flag_lgbm=True
clfs = list()<init_hyperparams>
df = df[df["Date"] >= df["Date"].min() + timedelta(days=days_shift[NUM_SHIFT])].copy()
for col in TARGETS:
df[col] = np.log1p(df[col])/normfactor
df = df[df['days']>TRAIN_START_DAY]
params = {
'num_leaves': 355,
'max_bin': 256+128+64,
'feature_fraction': 0.548,
'bagging_fraction': 0.648,
'bagging_freq': 11,
'min_data_in_leaf': 31+64+8,
'max_depth': 15+4,
'objective': 'binary',
'learning_rate': 0.0212345,
"boosting_type": "gbdt",
"metric": 'auc',
'seed': 123
}<merge>
def nn_block(input_layer, size, dropout_rate, activation):
out_layer = KL.Dense(size, activation=None )(input_layer)
 out_layer = KL.Activation(activation )(out_layer)
 out_layer = KL.Dropout(dropout_rate )(out_layer)
 return out_layer
def get_model(feature_length,target_length,):
inp = KL.Input(shape=(feature_length,))
hidden_layer = nn_block(inp, 64, 0.0, "relu")
 gate_layer = nn_block(hidden_layer, 32, 0.0, "sigmoid")
 hidden_layer = nn_block(hidden_layer, 64, 0.0, "relu")
 hidden_layer = nn_block(hidden_layer, 32, 0.0, "relu")
 hidden_layer = KL.multiply([hidden_layer, gate_layer])
 out = KL.Dense(target_length, activation="linear" )(hidden_layer)
 model = tf.keras.models.Model(inputs=[inp], outputs=out)
 return model
def get_input(df,features):
return [df[features]]
def train_models(df,features,targets,save=False):
print()
print("TRAINING.Der regnes på denne model:")
 get_model(len(features),len(targets)).summary()
models = []
for i in range(NUM_MODELS):
print("PHS1")
 model = get_model(len(features),len(targets))
print("PHS2")
 model.compile(loss="mean_squared_error", optimizer=Nadam(lr=1e-4))
print("PHS3 - Targets:",targets)
 hist = model.fit(get_input(df,features), df[targets],
batch_size=2048, epochs=500, verbose=0, shuffle=True)
 print("PHS4")
 if save:
print("PHS5")
 model.save_weights("model{}.h5".format(i))
print("PHS6")
 models.append(model)
 print("PHS7")
 return models
trains=list()
valids=list()
num=1
for i in range(0,num):
train_df_clf=train_df[1200*10000:2*1200*10000]
print('sample end')
 del train_df
users=train_df_clf['user_id'].drop_duplicates()
users=users.sample(frac=0.08, random_state=42)
 users_df=pd.DataFrame()
users_df['user_id']=users.values
valid_df_newuser = pd.merge(train_df_clf, users_df, on=['user_id'], how='inner',right_index=True)
 del users_df
del users
gc.collect()
train_df_clf.drop(valid_df_newuser.index, inplace=True)
 print('pd.merge(train_df_clf, questions_df)')
 train_df_clf = pd.merge(train_df_clf, questions_df, on='content_id', how='left',right_index=True)
 valid_df_newuser = pd.merge(valid_df_newuser, questions_df, on='content_id', how='left',right_index=True)
 print('valid_df')
 valid_df = train_df_clf.sample(frac=0.03, random_state=42)
 train_df_clf.drop(valid_df.index, inplace=True)
 valid_df = valid_df.append(valid_df_newuser)
 del valid_df_newuser
gc.collect()
trains.append(train_df_clf)
 valids.append(valid_df)
 print('train_df_clf length：',len(train_df_clf))
print('valid_df length：',len(valid_df))<set_options>
def rmse(y_true, y_pred):
return np.sqrt(mean_squared_error(y_true, y_pred))
def evaluate(df,targets):
error = 0
for col in targets:
error += rmse(df[col].values, df["pred_{}".format(col)].values)
 return np.round(error/len(targets), 5)
 def predict_one(df,features,prev_targets, models):
pred = np.zeros(( df.shape[0], 2))
for model in models:
pred += model.predict(get_input(df,features)) /len(models)
 if np.isnan(np.sum(pred)) :
print("DYYYYYYYYYYYYT: der er NAN")
 print(pred)
 pred = np.maximum(pred, df[prev_targets].values)
 pred[:, 0] = np.log1p(np.expm1(normfactor*pred[:, 0])+ 0.1)/normfactor
pred[:, 1] = np.log1p(np.expm1(normfactor*pred[:, 1])+ 0.01)/normfactor
if np.isnan(np.sum(pred)) :
print("DYT DYT DYT: der er NAN")
 return np.clip(pred, None, 15)
 def predict(test_df,features,targets,prev_targets,first_day, num_days, models, val=False):
for d in range(0, num_days):
print("DAY NO.",d)
 test_df = fill_shift_columns(test_df,targets)
 date = first_day + timedelta(days=d)
 temp_df = test_df.loc[test_df["Date"] == date].copy()
y_pred = predict_one(temp_df,features,prev_targets, models)
 for i, col in enumerate(targets):
test_df.loc[test_df["Date"] == date, col] = y_pred[:, i]
if add_other:
if USE_NEW:
test_df.loc[test_df["Date"] == date,"NewCases_ppm"] = \
np.log1p(np.expm1(y_pred[:, 0])*1000000/ \
np.expm1(test_df.loc[test_df["Date"] == date,"Population(2020)"]))
test_df.loc[test_df["Date"] == date,"NewFatalities_ppm"] = \
np.log1p(np.expm1(y_pred[:, 1])*1000000/ \
np.expm1(test_df.loc[test_df["Date"] == date,"Population(2020)"]))
else:
test_df.loc[test_df["Date"] == date,"ConfirmedCases_ppm"] = \
np.log1p(np.expm1(y_pred[:, 0])*1000000/ \
np.expm1(test_df.loc[test_df["Date"] == date,"Population(2020)"]))
test_df.loc[test_df["Date"] == date,"Fatalities_ppm"] = \
np.log1p(np.expm1(y_pred[:, 1])*1000000/ \
np.expm1(test_df.loc[test_df["Date"] == date,"Population(2020)"]))
if val:
print(evaluate(test_df[test_df["Date"] == date]))
return test_df
del train_df_clf
del valid_df
gc.collect()<prepare_x_and_y>
all_features = base_features + shift_features
df = fill_shift_columns(df,TARGETS)
 df[all_features] = df[all_features].fillna(0)
 print("Kolonner i modelller",all_features)
 print("BEFORE TRAINING")
 print(df[(df['Country_Region']=='Germany')&(df['days'] >75)])
 final_models = train_models(df,all_features,TARGETS, save=True )
for i in range(0,num):
X_train_np = trains[i][features].values.astype(np.float32)
 X_valid_np = valids[i][features].values.astype(np.float32)
 tr_data = lgb.Dataset(X_train_np, label=trains[i][target], feature_name=list(features))
va_data = lgb.Dataset(X_valid_np, label=valids[i][target], feature_name=list(features))
del trains
del valids
del X_train_np
del X_valid_np
gc.collect()
model = lgb.train(
 params,
tr_data,
num_boost_round=7000,
valid_sets=[tr_data, va_data],
early_stopping_rounds=50,
feature_name=features,
categorical_feature=categorical_columns,
verbose_eval=50 
)
 clfs.append(model)
 fig,ax = plt.subplots(figsize=(15,15))
lgb.plot_importance(model, ax=ax,importance_type='gain',max_num_features=50)
 plt.show()
del tr_data
del va_data
gc.collect()<import_modules>
full_df_pred= predict(full_df,all_features,TARGETS,prev_targets,ESTIMATE_FIRST_DATE,ESTIMATE_DAYS, final_models)
 for col in TARGETS:
full_df_pred[col] = np.expm1(full_df_pred[col])
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader<define_variables>
gem_targets = gem_targets[gem_targets["Date"]>=SUBMISSION_FIRST_DATE]
print(gem_targets.head(15))
values_to_submit = full_df_pred[full_df_pred["Date"]>=ESTIMATE_FIRST_DATE]
values_to_submit = values_to_submit[['Date','Country_Region','Province_State','ConfirmedCases', 'Fatalities']]
print(values_to_submit.head(15))
values_to_submit = gem_targets.append(values_to_submit, sort=False)
 print(values_to_submit.tail(10))
print(sub_df[['Date','Country_Region','Province_State','ForecastId']])
 values_to_submit = pd.merge(values_to_submit,sub_df[['Date','Country_Region','Province_State','ForecastId']], on=['Date','Country_Region','Province_State'], how='left')
 print(values_to_submit.head(20))
print(values_to_submit.tail(20))
sub2 = values_to_submit[["ForecastId"] + TARGETS]
sub2["ForecastId"] = sub2["ForecastId"].astype(np.int16)
MAX_SEQ = 240
ACCEPTED_USER_CONTENT_SIZE = 2
EMBED_SIZE = 256
BATCH_SIZE = 64+32
DROPOUT = 0.1<choose_model_class>
sub2.sort_values("ForecastId", inplace=True)
 sub2.to_csv("submission.csv", index=False)
class FFN(nn.Module):
def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):
super(FFN, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.bn = nn.BatchNorm1d(bn_size)
 self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.dropout = nn.Dropout(dropout)
 def forward(self, x):
x = self.relu(self.lr1(x))
x = self.bn(x)
 x = self.lr2(x)
 return self.dropout(x)
 class FFN0(nn.Module):
def __init__(self, state_size = 200, forward_expansion = 1, bn_size = MAX_SEQ - 1, dropout=0.2):
super(FFN0, self ).__init__()
self.state_size = state_size
self.lr1 = nn.Linear(state_size, forward_expansion * state_size)
 self.relu = nn.ReLU()
self.lr2 = nn.Linear(forward_expansion * state_size, state_size)
 self.layer_normal = nn.LayerNorm(state_size)
 self.dropout = nn.Dropout(0.2)
 def forward(self, x):
x = self.lr1(x)
 x = self.relu(x)
 x = self.lr2(x)
 x=self.layer_normal(x)
 return self.dropout(x )<categorify>
full_df_pred['Cases_Estimate'] = full_df_pred['ConfirmedCases']
full_df_pred['Fatalities_Estimate'] = full_df_pred['Fatalities']
full_df_pred2 = full_df_pred[['Date','Country_Region','Province_State']+TARGETS]
full_df2 = full_df[['Date','Country_Region','Province_State']+TARGETS]
full_df3 = pd.merge(full_df2,full_df_pred[['Date','Country_Region','Province_State','Cases_Estimate','Fatalities_Estimate']], on=['Date','Country_Region','Province_State'], how='left')
 mask = full_df3['Cases_Estimate'].notna()
full_df3.loc[mask,'CasesConfirmed'] = full_df3.loc[mask,'Cases_Estimate']
full_df3.loc[mask,'Fatalities'] = full_df3.loc[mask,'Fatalities_Estimate']
split_on = 'Country_Region'
split_values = full_df3[split_on].unique()
kolonne = 'CasesConfirmed'
for imin in range(0,175,25):
plt.figure(figsize=(30,30))
imax = imin+25
for i in range(imin,imax):
plt.subplot(5,5,i-imin+1)
 idx = i
df_interest = full_df3[full_df3[split_on]==split_values[idx]].reset_index(drop=True)
 tmp = df_interest[kolonne].values
sns.lineplot(x=df_interest['Date'], y=tmp, label='pred')
 df_interest2 = full_df3[(full_df3[split_on]==split_values[idx])&(full_df3['Date']<=GIVEN_LAST_DATE)].reset_index(drop=True)
 sns.lineplot(x=df_interest2['Date'].values, y=df_interest2[kolonne].values, label='true')
 plt.title(split_on+' '+str(split_values[idx]))
plt.show()