Kaggle competition: Challenge GH
names=[]
for day in days_back_confimed:
names+=["days_ago_confirmed_count_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
for day in days_back_fatalities:
names+=["days_ago_fatalitiescount_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["confirmed_level"]
names+=["fatalities_level"]
if not extra_stable_columns is None and len(extra_stable_columns)>0:
names+=[k for k in extra_stable_columns]
if not group_names is None:
for gg in range(len(group_names)) :
for day in days_back_confimed_group:
names+=["days_ago_grooupcount_" + str(gg+1)+ "_" + str(day)]
for window in windows_group:
names+=["ma_group_" + str(gg+1)+ "_" + str(window)+ "_rate_" + str(k+1)for k in range(size_group)] 
<define_variables>
%matplotlib inline
plt.style.use('seaborn-whitegrid')
 warnings.filterwarnings('ignore' )
tr_frame=train_frame
target_confirmed=["confirmed_plus" + str(k+1)for k in range(horizon)]
target_fatalities=["fatalities_plus" + str(k+1)for k in range(horizon)]
seed=1412
target_confirmed_train=tr_frame[target_confirmed].values
print(" original shape of train is {} ".format(target_confirmed_train.shape))
target_fatalities_train=tr_frame[target_fatalities].values
features_train=tr_frame[names].values
standard_confirmed_train=tr_frame["ConfirmedCases"].values
standard_fatalities_train=tr_frame["Fatalities"].values
current_confirmed_train=tr_frame["ConfirmedCases"].values
features_cv=[]
name_cv=[]
standard_confirmed_cv=[]
standard_fatalities_cv=[]
names_=tr_frame["key"].values
training_horizon=int(features_train.shape[0]/len(unique_keys))
print("training horizon = ",training_horizon)
 for dd in range(training_horizon-1,features_train.shape[0],training_horizon):
features_cv.append(features_train[dd])
 name_cv.append(names_[dd])
 standard_confirmed_cv.append(standard_confirmed_train[dd])
 standard_fatalities_cv.append(standard_fatalities_train[dd])
 print(name_cv[-1], standard_confirmed_cv[-1], standard_fatalities_cv[-1])
 current_confirmed_train=[k for k in range(len(current_confirmed_train)) if current_confirmed_train[k]>0]
target_confirmed_train=target_confirmed_train[current_confirmed_train]
target_fatalities_train=target_fatalities_train[current_confirmed_train]
features_train=features_train[current_confirmed_train]
standard_confirmed_train=standard_confirmed_train[current_confirmed_train]
standard_fatalities_train=standard_fatalities_train[current_confirmed_train]
features_cv=np.array(features_cv)
 preds_confirmed_cv=np.zeros(( features_cv.shape[0],horizon))
preds_confirmed_standard_cv=np.zeros(( features_cv.shape[0],horizon))
preds_fatalities_cv=np.zeros(( features_cv.shape[0],horizon))
preds_fatalities_standard_cv=np.zeros(( features_cv.shape[0],horizon))
overal_rmsle_metric_confirmed=0.0
for j in range(preds_confirmed_cv.shape[1]):
this_target=target_confirmed_train[:,j]
index_positive=[k for k in range(len(this_target)) if this_target[k]!=-1]
this_features=features_train[index_positive]
this_target=this_target[index_positive]
this_weight=np.log(standard_confirmed_train[index_positive]+3.)
this_features_cv=features_cv
preds=bagged_set_train(this_features,this_target,this_weight, seed, 1,features_cv, xt=None,yt=None, output_name=model_directory +"confirmed"+ str(j))
preds_confirmed_cv[:,j]=preds
print(" modelling confirmed, case %d, original train %d, and after %d, original cv %d and after %d "%(
 j,target_confirmed_train.shape[0],this_target.shape[0],this_features_cv.shape[0],this_features_cv.shape[0]))
predictions=[]
for ii in range(preds_confirmed_cv.shape[0]):
current_prediction=standard_confirmed_cv[ii]
if current_prediction==0 :
current_prediction=0.1
for j in range(preds_confirmed_cv.shape[1]):
current_prediction*=max(1,preds_confirmed_cv[ii][j])
 preds_confirmed_standard_cv[ii][j]=current_prediction
for j in range(preds_confirmed_cv.shape[1]):
this_target=target_fatalities_train[:,j]
index_positive=[k for k in range(len(this_target)) if this_target[k]!=-1]
this_features=features_train[index_positive]
this_target=this_target[index_positive]
this_weight=np.log(standard_confirmed_train[index_positive]+2.)
this_features_cv=features_cv
preds=bagged_set_train(this_features,this_target,this_weight, seed, 1,features_cv, xt=None,yt=None, output_name=model_directory +"fatal"+ str(j))
preds_fatalities_cv[:,j]=preds
print(" modelling fatalities, case %d, original train %d, and after %d, original cv %d and after %d "%(
 j,target_confirmed_train.shape[0],this_target.shape[0],this_features_cv.shape[0],this_features_cv.shape[0]))
predictions=[]
for ii in range(preds_fatalities_cv.shape[0]):
current_prediction=standard_fatalities_cv[ii]
if current_prediction==0 and standard_confirmed_cv[ii]>400:
current_prediction=0.1
for j in range(preds_fatalities_cv.shape[1]):
if current_prediction==0 and preds_confirmed_standard_cv[ii][j]>400:
current_prediction=1.
 current_prediction*=max(1,preds_fatalities_cv[ii][j])
 preds_fatalities_standard_cv[ii][j]=current_prediction 
<categorify>
train_data = pd.read_csv('.. /input/challenge-gh/TrainData.csv', encoding='ISO-8859-1',sep=';')
 test_data = pd.read_csv('.. /input/challenge-gh/TestData.csv', encoding='ISO-8859-1',sep=';')
 submission_data = pd.read_csv('.. /input/challenge-gh/Lsungstemplate.csv', encoding='ISO-8859-1' )
def decay_4_first_10_then_1_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
if j<10:
arr[j]=1.+(max(1,array[j])-1.) /4.
 else :
arr[j]=1.
 return arr
	
def decay_16_first_10_then_1_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
if j<10:
arr[j]=1.+(max(1,array[j])-1.) /16.
 else :
arr[j]=1.
 return arr	
def decay_2_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /2.
 return arr
def decay_4_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /4.
 return arr 	
	
def acceleratorx2_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) *2.
 return arr
def decay_1_5_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /1.5
return arr
def decay_1_2_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /1.2
return arr
def decay_1_1_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /1.1
return arr
def stay_same_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.
 return arr
def decay_2_last_12_linear_inter_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /2.
 arr12=(max(1,arr[-12])-1.) /12.
 for j in range(0, 12):
arr[len(arr)-12 +j]= max(1, 1 +(( arr12*12)-(j+1)*arr12))
return arr
def decay_1_2_last_12_linear_inter_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /1.2
arr12=(max(1,arr[-12])-1.) /12.
 for j in range(0, 12):
arr[len(arr)-12 +j]= max(1, 1 +(( arr12*12)-(j+1)*arr12))
return arr
def decay_1_5_last_12_linear_inter_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /1.5
arr12=(max(1,arr[-12])-1.) /12.
 for j in range(0, 12):
arr[len(arr)-12 +j]= max(1, 1 +(( arr12*12)-(j+1)*arr12))
return arr
def decay_4_last_12_linear_inter_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /4.
 arr12=(max(1,arr[-12])-1.) /12.
 for j in range(0, 12):
arr[len(arr)-12 +j]= max(1, 1 +(( arr12*12)-(j+1)*arr12))
return arr
def decay_8_last_12_linear_inter_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /8.
 arr12=(max(1,arr[-12])-1.) /12.
 for j in range(0, 12):
arr[len(arr)-12 +j]= max(1, 1 +(( arr12*12)-(j+1)*arr12))
return arr
def linear_last_12_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=max(1,array[j])
 arr12=(max(1,arr[-12])-1.) /12.
 for j in range(0, 12):
arr[len(arr)-12 +j]= max(1, 1 +(( arr12*12)-(j+1)*arr12))
return arr
decay_4_first_10_then_1 =[ "Jilin_China","Tianjin_China"]
decay_4_first_10_then_1_fatality=[]
decay_16_first_10_then_1 =["Beijing_China","Fujian_China","Jiangsu_China"]
decay_16_first_10_then_1_fatality=[]
decay_4=["nan_Central African Republic","Shanghai_China","nan_Korea, South","nan_Zambia"]
decay_4_fatality=["Heilongjiang_China","Inner Mongolia_China"]
decay_2 =["nan_Burundi","Faroe Islands_Denmark","nan_Eritrea","nan_Italy","nan_Brunei""Heilongjiang_China","Inner Mongolia_China",
"Shanxi_China","nan_Congo(Kinshasa)","nan_Cyprus","Reunion_France","Sint Maarten_Netherlands","nan_Angola"]
decay_2_fatality=["nan_Iran"]
stay_same=["nan_Diamond Princess"]
stay_same_fatality=["Beijing_China","Fujian_China","Qinghai_China","Jiangsu_China","Jilin_China","Macau_China","Ningxia_China","Tianjin_China"
,"Faroe Islands_Denmark","Shanxi_China"]
normal=[]
normal_fatality=["nan_Andorra","New South Wales_Australia","nan_Korea, South","New York_US","nan_Austria","Alberta_Canada","Nova Scotia_Canada",
"Saskatchewan_Canada","nan_Costa Rica","nan_Croatia","nan_Finland","Mayotte_France","nan_Hungary","nan_Iceland","nan_Latvia","nan_Czechia"
,"nan_Luxembourg","nan_Malta","nan_Montenegro","nan_New Zealand","nan_Norway","nan_South Africa","Hawaii_US","Ohio_US","Pennsylvania_US","nan_Uruguay","nan_Venezuela",
"nan_West Bank and Gaza"]
decay_4_last_12_linear_inter =[ "nan_Dominica","New Caledonia_France",
"Saint Barthelemy_France","nan_Gambia","nan_Grenada","nan_Holy See",
"nan_Mauritania","nan_Nicaragua","nan_Saint Lucia","nan_Saint Vincent and the Grenadines",
"nan_Seychelles","nan_Suriname","nan_Mongolia","nan_Cabo Verde",
"Montserrat_United Kingdom","Turks and Caicos Islands_United Kingdom", "Hong Kong_China","Northern Territory_Australia","Northwest Territories_Canada","Yukon_Canada",
"Guangdong_China","nan_MS Zaandam","nan_Maldives","nan_Saint Kitts and Nevis","nan_Sao Tome and Principe","nan_South Sudan","nan_Western Sahara"]
decay_4_last_12_linear_inter_fatality=["nan_Somalia"]
decay_2_last_12_linear_inter =[ "nan_Chad","French Polynesia_France","nan_Laos","nan_Nepal","nan_Sudan","Bermuda_United Kingdom","Cayman Islands_United Kingdom"
,"nan_Eswatini","nan_Guyana","nan_Liberia","nan_Malawi","nan_Mozambique","nan_Timor-Leste",
"Falkland Islands(Malvinas)_United Kingdom","nan_Equatorial Guinea"]
decay_2_last_12_linear_inter_fatality=[]
decay_1_5 =["nan_Cambodia","nan_Croatia","nan_Denmark",
"nan_Bahamas","nan_Bahrain","nan_Barbados" ,"Manitoba_Canada","New Brunswick_Canada",
"Saskatchewan_Canada","nan_France","nan_Libya","nan_Malta",
	"Aruba_Netherlands","nan_Niger","nan_Spain","Virgin Islands_US",
	"Channel Islands_United Kingdom","Gibraltar_United Kingdom","nan_United Kingdom",'nan_Burma',"nan_Congo(Brazzaville)",
	"French Guiana_France","Martinique_France","Mayotte_France","nan_Georgia","nan_Iran","nan_New Zealand",
	"nan_West Bank and Gaza"]
decay_1_5_fatality=["nan_Cuba","nan_Morocco","Ohio_US","Pennsylvania_US","Puerto Rico_US" ,"nan_Uzbekistan",
"nan_Cyprus","nan_France","nan_Niger","South Dakota_US","nan_Gabon"]
acceleratorx2=[]
acceleratorx2_fatality=["Newfoundland and Labrador_Canada"]
decay_1_5_last_12_linear_inter =["nan_Antigua and Barbuda","nan_Botswana","nan_Haiti","nan_Madagascar","nan_Somalia",
"nan_Zimbabwe"]
decay_1_5_last_12_linear_inter_fatality =["nan_Burma","nan_Cabo Verde","nan_Djibouti"]
decay_1_2 =["nan_Albania","nan_Andorra","New South Wales_Australia","nan_Austria","nan_Azerbaijan","nan_Bangladesh","nan_Belize","Alberta_Canada","nan_Mauritius","nan_Monaco","nan_Montenegro",
"Newfoundland and Labrador_Canada","nan_Costa Rica","nan_Czechia","nan_Dominican Republic","nan_Estonia","Guadeloupe_France","nan_Iceland","nan_Latvia","nan_Liechtenstein","nan_Luxembourg","nan_Norway",
"nan_Poland","nan_Rwanda","nan_South Africa","nan_Trinidad and Tobago","Arizona_US","California_US","Colorado_US","Connecticut_US","Georgia_US","Hawaii_US","Maine_US","New York_US","Oklahoma_US",
"Oregon_US","Pennsylvania_US","Tennessee_US","Washington_US","nan_Uruguay","nan_Venezuela","nan_Vietnam",
"nan_Gabon","nan_Syria","nan_Djibouti"]
decay_1_2_fatality=["nan_Afghanistan","nan_Belarus","nan_Mali","nan_Mexico","nan_Peru","nan_Serbia","Kentucky_US","Puerto Rico_US","Texas_US","nan_Ukraine"]
decay_1_1 =["nan_Algeria","nan_Argentina","Tasmania_Australia","Nova Scotia_Canada","nan_Finland","nan_Hungary","nan_Iraq",
"nan_Jordan","nan_Kyrgyzstan","nan_Malaysia","Alabama_US","Iowa_US","Michigan_US","Missouri_US","Nebraska_US","North Dakota_US ","Ohio_US","South Carolina_US"]
decay_1_1_fatality=["nan_Ecuador","nan_Kenya","nan_Korea, South","nan_Kosovo","Indiana_US","New Mexico_US","Oregon_US","Rhode Island_US","Tennessee_US","nan_United Arab Emirates"]
decay_1_2_last_12_linear_inter =[]
decay_1_2_last_12_linear_inter_fatality=["nan_Angola" ]
linear_last_12=["nan_Afghanistan","nan_Ireland","nan_Benin","nan_Tanzania"]
linear_last_12_fatality=["nan_Guatemala","nan_Ireland"]
decay_8_last_12_linear_inter =[ "nan_Bhutan","Prince Edward Island_Canada","Greenland_Denmark","nan_Fiji","Saint Pierre and Miquelon_France","St Martin_France","nan_Namibia",
"Bonaire, Sint Eustatius and Saba_Netherlands","Curacao_Netherlands","nan_Papua New Guinea","Anguilla_United Kingdom","British Virgin Islands_United Kingdom",]
decay_8_last_12_linear_inter_fatality=[]
tr_frame=train_frame
features_train=tr_frame[names].values
standard_confirmed_train=tr_frame["ConfirmedCases"].values
standard_fatalities_train=tr_frame["Fatalities"].values
current_confirmed_train=tr_frame["ConfirmedCases"].values
features_cv=[]
name_cv=[]
standard_confirmed_cv=[]
standard_fatalities_cv=[]
names_=tr_frame["key"].values
training_horizon=int(features_train.shape[0]/len(unique_keys))
print("training horizon = ",training_horizon)
 for dd in range(training_horizon-1,features_train.shape[0],training_horizon):
features_cv.append(features_train[dd])
 name_cv.append(names_[dd])
 standard_confirmed_cv.append(standard_confirmed_train[dd])
 standard_fatalities_cv.append(standard_fatalities_train[dd])
 print(name_cv[-1], standard_confirmed_cv[-1], standard_fatalities_cv[-1])
 features_cv=np.array(features_cv)
 preds_confirmed_cv=np.zeros(( features_cv.shape[0],horizon))
preds_confirmed_standard_cv=np.zeros(( features_cv.shape[0],horizon))
preds_fatalities_cv=np.zeros(( features_cv.shape[0],horizon))
preds_fatalities_standard_cv=np.zeros(( features_cv.shape[0],horizon))
overal_rmsle_metric_confirmed=0.0
for j in range(preds_confirmed_cv.shape[1]):
this_features_cv=features_cv
preds=predict(features_cv, input_name=model_directory +"confirmed"+ str(j))
preds_confirmed_cv[:,j]=preds
print(" modelling confirmed, case %d, , original cv %d and after %d "%(j,this_features_cv.shape[0],this_features_cv.shape[0]))
predictions=[]
for ii in range(preds_confirmed_cv.shape[0]):
current_prediction=standard_confirmed_cv[ii]
if current_prediction==0 :
current_prediction=0.1
this_preds=preds_confirmed_cv[ii].tolist()
name=name_cv[ii]
reserve=this_preds[0]
if name in normal:
this_preds=this_preds	
	
elif name in decay_4_first_10_then_1:
this_preds=decay_4_first_10_then_1_f(this_preds)
 		
elif name in decay_16_first_10_then_1:
this_preds=decay_16_first_10_then_1_f(this_preds)		
		
elif name in decay_4_last_12_linear_inter:
this_preds=decay_4_last_12_linear_inter_f(this_preds)		
elif name in decay_8_last_12_linear_inter:
this_preds=decay_8_last_12_linear_inter_f(this_preds)		
elif name in decay_4:
this_preds=decay_4_f(this_preds)
 elif name in decay_2:
this_preds=decay_2_f(this_preds)
 elif name in decay_2_last_12_linear_inter:
this_preds=decay_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_5:
this_preds=decay_1_5_f(this_preds)
 elif name in decay_1_5_last_12_linear_inter:
this_preds=decay_1_5_last_12_linear_inter_f(this_preds)
 elif name in decay_1_2:
this_preds=decay_1_2_f(this_preds)
 elif name in decay_1_2_last_12_linear_inter:
this_preds=decay_1_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_1:
this_preds=decay_1_1_f(this_preds)
 elif name in linear_last_12:
this_preds=linear_last_12_f(this_preds)
 elif name in acceleratorx2:
this_preds=acceleratorx2_f(this_preds)
 elif name in stay_same or "China" in name:
this_preds=stay_same_f(this_preds)
 for j in range(preds_confirmed_cv.shape[1]):
if j==0 and "nan_Congo(Brazzaville)" in name:
current_prediction=117.
 if j==0 and "nan_Cabo Verde" in name:
current_prediction=56.
 current_prediction*=max(1,this_preds[j])
 preds_confirmed_standard_cv[ii][j]=current_prediction
for j in range(preds_confirmed_cv.shape[1]):
this_features_cv=features_cv
preds=predict(features_cv, input_name=model_directory +"fatal"+ str(j))
preds_fatalities_cv[:,j]=preds
print(" modelling fatalities, case %d, original cv %d and after %d "%(j,this_features_cv.shape[0],this_features_cv.shape[0]))
predictions=[]
for ii in range(preds_fatalities_cv.shape[0]):
current_prediction=standard_fatalities_cv[ii]
this_preds=preds_fatalities_cv[ii].tolist()
name=name_cv[ii]
reserve=this_preds[0]
if name in normal_fatality:
this_preds=this_preds	 	
	
elif name in decay_4_first_10_then_1_fatality:
this_preds=decay_4_first_10_then_1_f(this_preds)
 		
elif name in decay_16_first_10_then_1_fatality:
this_preds=decay_16_first_10_then_1_f(this_preds)
 elif name in decay_4_last_12_linear_inter_fatality:
this_preds=decay_4_last_12_linear_inter_f(this_preds)		
elif name in decay_8_last_12_linear_inter_fatality:
this_preds=decay_8_last_12_linear_inter_f(this_preds)		
elif name in decay_4_fatality:
this_preds=decay_4_f(this_preds)		
		
elif name in decay_2_fatality:
this_preds=decay_2_f(this_preds)
 elif name in decay_2_last_12_linear_inter_fatality:
this_preds=decay_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_5_fatality:
this_preds=decay_1_5_f(this_preds)	
elif name in decay_1_5_last_12_linear_inter_fatality:
this_preds=decay_1_5_last_12_linear_inter_f(this_preds)
 elif name in decay_1_2_fatality:
this_preds=decay_1_2_f(this_preds)
 elif name in decay_1_2_last_12_linear_inter_fatality:
this_preds=decay_1_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_1_fatality:
this_preds=decay_1_1_f(this_preds)
 elif name in linear_last_12_fatality:
this_preds=linear_last_12_f(this_preds)
 elif name in acceleratorx2_fatality:
this_preds=acceleratorx2_f(this_preds)
 elif name in stay_same_fatality:
this_preds=stay_same_f(this_preds)
 elif name in normal:
this_preds=this_preds	
	
elif name in decay_4_first_10_then_1:
this_preds=decay_4_first_10_then_1_f(this_preds)
 		
elif name in decay_16_first_10_then_1:
this_preds=decay_16_first_10_then_1_f(this_preds)		
elif name in decay_4_last_12_linear_inter:
this_preds=decay_4_last_12_linear_inter_f(this_preds)		
elif name in decay_8_last_12_linear_inter:
this_preds=decay_8_last_12_linear_inter_f(this_preds)		
elif name in decay_4:
this_preds=decay_4_f(this_preds)
 		
elif name in decay_2:
this_preds=decay_2_f(this_preds)
 elif name in decay_2_last_12_linear_inter:
this_preds=decay_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_5:
this_preds=decay_1_5_f(this_preds)
 elif name in decay_1_5_last_12_linear_inter:
this_preds=decay_1_5_last_12_linear_inter_f(this_preds)
 elif name in decay_1_2:
this_preds=decay_1_2_f(this_preds)
 elif name in decay_1_2_last_12_linear_inter:
this_preds=decay_1_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_1:
this_preds=decay_1_1_f(this_preds)
 elif name in linear_last_12:
this_preds=linear_last_12_f(this_preds)
 elif name in acceleratorx2:
this_preds=acceleratorx2_f(this_preds)
 elif name in stay_same or "China" in name:
this_preds=stay_same_f(this_preds)
 for j in range(preds_fatalities_cv.shape[1]):
if current_prediction==0 and preds_confirmed_standard_cv[ii][j]>400 and(name not in stay_same and name not in stay_same_fatality and "China" not in name):
current_prediction=1.
 if j==0 and "Aruba_Netherlands" in name:
current_prediction=1.
 if j==0 and "nan_Monaco" in name:
current_prediction=3.
 if j==0 and 'nan_Costa Rica' in name or "Isle of Man_United Kingdom" in name:
current_prediction=4.
 if j==0 and "nan_Somalia" in name:
current_prediction=5
if j==0 and "Martinique_France" in name:
current_prediction=8.
 current_prediction*=max(1,this_preds[j])
 preds_fatalities_standard_cv[ii][j]=current_prediction 
<define_variables>
df_train = train_data.replace('Unbekannt', np.nan, inplace=False )
key_to_confirmed_rate={}
key_to_fatality_rate={}
key_to_confirmed={}
key_to_fatality={}
print(len(features_cv), len(name_cv),len(standard_confirmed_cv),len(standard_fatalities_cv))
print(preds_confirmed_cv.shape,preds_confirmed_standard_cv.shape,preds_fatalities_cv.shape,preds_fatalities_standard_cv.shape)
 for j in range(len(name_cv)) :
key_to_confirmed_rate[name_cv[j]]=preds_confirmed_cv[j,:].tolist()
key_to_fatality_rate[name_cv[j]]=preds_fatalities_cv[j,:].tolist()
key_to_confirmed[name_cv[j]] =preds_confirmed_standard_cv[j,:].tolist()
key_to_fatality[name_cv[j]]=preds_fatalities_standard_cv[j,:].tolist() 
<merge>
def encode_categorical_features(df):

 lb_make = LabelEncoder()
obj_df = df.select_dtypes(include=['object'] ).copy()
cat_col_names=obj_df.columns.tolist()
for name in cat_col_names:
df[name] = lb_make.fit_transform(df[name])
 return df
train_new=train[["Date","ConfirmedCases","Fatalities","key","rate_ConfirmedCases","rate_Fatalities"]]
test_new=pd.merge(test,train_new, how="left", left_on=["key","Date"], right_on=["key","Date"] ).reset_index(drop=True)
 test_new<categorify>
def fillin_columns(frame,key_column, original_name, training_horizon, test_horizon, unique_values, key_to_values):
keys=frame[key_column].values
original_values=frame[original_name].values.tolist()
print(len(keys), len(original_values), training_horizon ,test_horizon,len(key_to_values))
for j in range(unique_values):
current_index=(j *(training_horizon +test_horizon)) +training_horizon
current_key=keys[current_index]
values=key_to_values[current_key]
co=0
for g in range(current_index, current_index + test_horizon):
original_values[g]=values[co]
co+=1
frame[original_name]=original_values
all_days=int(test_new.shape[0]/len(unique_keys))
tr_horizon=all_days-horizon
print(all_days,tr_horizon, horizon)
 fillin_columns(test_new,"key", 'ConfirmedCases', tr_horizon, horizon, len(unique_keys), key_to_confirmed)
 fillin_columns(test_new,"key", 'Fatalities', tr_horizon, horizon, len(unique_keys), key_to_fatality)
 fillin_columns(test_new,"key", 'rate_ConfirmedCases', tr_horizon, horizon, len(unique_keys), key_to_confirmed_rate)
 fillin_columns(test_new,"key", 'rate_Fatalities', tr_horizon, horizon, len(unique_keys), key_to_fatality_rate )<categorify>
columns_to_drop=columns_to_drop = ['Stammnummer', 'Anruf-ID','Tage seit letzter Kampagne','Ergebnis letzte Kampagne','Tag','Alter','Kontostand','Anzahl der Ansprachen','Anzahl Kontakte letzte Kampagne']
df_train=df_train.drop(columns_to_drop,axis=1)
 df_train = df_train.apply(lambda x: x.fillna(x.value_counts().index[0]))
df_train=encode_categorical_features(df_train )
<train_model>
df_train[['Dauer']] = df_train[['Dauer']].apply(np.sqrt)
 scaler = MinMaxScaler(feature_range=(0, 1))
df_train[['Dauer']] = scaler.fit_transform(df_train[['Dauer']])
 train_features_all = df_train.drop(['Zielvariable'], axis=1, errors='ignore')
 train_labels_all =df_train.Zielvariable
def bagged_set_trainc(X_ts,y_cs,wts, seed, estimators,xtest, xt=None,yt=None, output_name=None):
baggedpred=np.array([ 0.0 for d in range(0, xtest.shape[0])])
 for n in range(0, estimators):
params = {'objective': 'rmse',
'metric': 'rmse',
'boosting': 'gbdt',
'learning_rate': 0.005,
'drop_rate':0.01,
'skip_drop':0.6,
'uniform_drop':True,
'verbose': -1,
'num_leaves': 30,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'bagging_seed': seed + n,
'feature_fraction': 0.8,
'feature_fraction_seed': seed + n,
'min_data_in_leaf': 10,
'max_bin': 100,
'max_depth':20,
'reg_alpha':1,
'lambda_l2': 10,
'num_threads':6
}
d_train = lgb.Dataset(X_ts,y_cs, weight=wts, free_raw_data=False)
 if not type(yt)is type(None):
d_cv = lgb.Dataset(xt,yt, free_raw_data=False, reference=d_train)
 model = lgb.train(params,d_train,num_boost_round=500,
valid_sets=d_cv,
verbose_eval=50)
 else :
model = lgb.train(params,d_train,num_boost_round=500)
 preds=model.predict(xtest)
 baggedpred+=preds
if not output_name is None:
joblib.dump(( model), output_name)

baggedpred/= estimators
return baggedpred 
<define_variables>
def hyper_parameter_tuning(clf_model,Xtrain,ytrain,parameters):

 precision_scorer = make_scorer(precision_score)
 grid_obj = GridSearchCV(estimator=clf_model, param_grid=parameters, scoring=precision_scorer,cv = 5, n_jobs = -1, verbose = 2)
 grid_obj = grid_obj.fit(Xtrain, ytrain)
 model = grid_obj.best_estimator_
return model
names=[]
for day in days_back_confimed:
names+=["days_ago_confirmed_count_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["ma" + str(window)+ "_count_confirmed" + str(k+1)for k in range(size)]
for day in days_back_fatalities:
names+=["days_ago_fatalitiescount_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["ma" + str(window)+ "_count_fatalities" + str(k+1)for k in range(size)]
names+=["confirmed_level"]
names+=["fatalities_level"]
if not extra_stable_columns is None and len(extra_stable_columns)>0:
names+=[k for k in extra_stable_columns]
if not group_names is None:
for gg in range(len(group_names)) :
for day in days_back_confimed_group:
names+=["days_ago_grooupcount_" + str(gg+1)+ "_" + str(day)]
for window in windows_group:
names+=["ma_group_" + str(gg+1)+ "_" + str(window)+ "_rate_" + str(k+1)for k in range(size_group)] 
<define_variables>
def recursiveFeature_elimination(df,model):

 features = df.drop('Zielvariable', axis=1)
 label = df['Zielvariable']
print('original dataset shape %s' % Counter(label))
sm = SMOTE(random_state=43,sampling_strategy='auto')
 features_resampled, labels_resampled =sm.fit_resample(features, label)
 print('Resampled dataset shape %s' % Counter(labels_resampled))
clf = model
RFE = RFECV(clf, step=1, cv=5,scoring='accuracy')
 RFE.fit(features_resampled,labels_resampled)
 support=RFE.support_
feature_names = np.array(features.columns.tolist())
 features_to_use= feature_names[support].tolist()
print(len(feature_names))
print(len(features_to_use))
print(features_to_use)
 return features_to_use
tr_frame=train_frame
if not train_frame_supplamanteary2 is None:
tr_frame=train_frame_supplamanteary2
target_confirmed=["confirmed_count_plus" + str(k+1)for k in range(horizon)]
target_fatalities=["fatalities_count_plus" + str(k+1)for k in range(horizon)]
seed=1412
target_confirmed_train=tr_frame[target_confirmed].values
print(" original shape of train is {} ".format(target_confirmed_train.shape))
target_fatalities_train=tr_frame[target_fatalities].values
features_train=tr_frame[names].values
standard_confirmed_train=tr_frame["ConfirmedCases"].values
standard_fatalities_train=tr_frame["Fatalities"].values
current_confirmed_train=tr_frame["ConfirmedCases"].values
features_cv=[]
name_cv=[]
standard_confirmed_cv=[]
standard_fatalities_cv=[]
names_=tr_frame["key"].values
training_horizon=int(features_train.shape[0]/len(unique_keys))
print("training horizon = ",training_horizon)
 for dd in range(training_horizon-1,features_train.shape[0],training_horizon):
features_cv.append(features_train[dd])
 name_cv.append(names_[dd])
 standard_confirmed_cv.append(standard_confirmed_train[dd])
 standard_fatalities_cv.append(standard_fatalities_train[dd])
 print(name_cv[-1], standard_confirmed_cv[-1], standard_fatalities_cv[-1])
 current_confirmed_train=[k for k in range(len(current_confirmed_train)) if current_confirmed_train[k]>0]
target_confirmed_train=target_confirmed_train[current_confirmed_train]
target_fatalities_train=target_fatalities_train[current_confirmed_train]
features_train=features_train[current_confirmed_train]
standard_confirmed_train=standard_confirmed_train[current_confirmed_train]
standard_fatalities_train=standard_fatalities_train[current_confirmed_train]
features_cv=np.array(features_cv)
 preds_confirmed_cv=np.zeros(( features_cv.shape[0],horizon))
preds_confirmed_standard_cv=np.zeros(( features_cv.shape[0],horizon))
preds_fatalities_cv=np.zeros(( features_cv.shape[0],horizon))
preds_fatalities_standard_cv=np.zeros(( features_cv.shape[0],horizon))
overal_rmsle_metric_confirmed=0.0
for j in range(preds_confirmed_cv.shape[1]):
this_target=target_confirmed_train[:,j]
index_positive=[k for k in range(len(this_target)) if this_target[k]!=-1]
this_features=features_train[index_positive]
this_target=this_target[index_positive]
this_weight=np.log(standard_confirmed_train[index_positive]+3.)
this_weight=[1.for k in range(len(this_weight)) ]
this_features_cv=features_cv
preds=bagged_set_trainc(this_features,this_target,this_weight, seed, 1,features_cv, xt=None,yt=None, output_name=model_directory +"confirmedc"+ str(j))
preds_confirmed_cv[:,j]=preds
print(" modelling count confirmed, case %d, original train %d, and after %d, original cv %d and after %d "%(
 j,target_confirmed_train.shape[0],this_target.shape[0],this_features_cv.shape[0],this_features_cv.shape[0]))
predictions=[]
for ii in range(preds_confirmed_cv.shape[0]):
current_prediction=standard_confirmed_cv[ii]
for j in range(preds_confirmed_cv.shape[1]):
current_prediction+=np.expm1(max(0,preds_confirmed_cv[ii][j]))
preds_confirmed_standard_cv[ii][j]=current_prediction
for j in range(preds_confirmed_cv.shape[1]):
this_target=target_fatalities_train[:,j]
index_positive=[k for k in range(len(this_target)) if this_target[k]!=-1]
this_features=features_train[index_positive]
this_target=this_target[index_positive]
this_weight=np.log(standard_confirmed_train[index_positive]+2.)
this_weight=[1.for k in range(len(this_weight)) ]
this_features_cv=features_cv
preds=bagged_set_trainc(this_features,this_target,this_weight, seed, 1,features_cv, xt=None,yt=None, output_name=model_directory +"fatalc"+ str(j))
preds_fatalities_cv[:,j]=preds
print(" modelling count fatalities, case %d, original train %d, and after %d, original cv %d and after %d "%(
 j,target_confirmed_train.shape[0],this_target.shape[0],this_features_cv.shape[0],this_features_cv.shape[0]))
predictions=[]
for ii in range(preds_fatalities_cv.shape[0]):
current_prediction=standard_fatalities_cv[ii]
for j in range(preds_fatalities_cv.shape[1]):
current_prediction+=np.expm1(max(0,preds_fatalities_cv[ii][j]))
preds_fatalities_standard_cv[ii][j]=current_prediction 
<categorify>
def run_kfold(model,X_all,y_all):

 kf = StratifiedKFold(n_splits=10, random_state=43, shuffle=True)
 Precisionscore_list = []
accuracyscore_list=[]
F1_score_list= []
fold = 0
for train_index, test_index in kf.split(X_all,y_all):
fold += 1
X_train, X_test = X_all.values[train_index], X_all.values[test_index]
y_train, y_test = y_all.values[train_index], y_all.values[test_index]
model.fit(X_train, y_train)
 predictions = model.predict(X_test)
 precisionscore = precision_score(y_test, predictions)
 accuracy = accuracy_score(y_test, predictions)
 F1score= f1_score(y_test,predictions)
 Precisionscore_list.append(precisionscore)
 accuracyscore_list.append(accuracy)
 F1_score_list.append(F1score)
 print("Fold {0} precision_score: {1}".format(fold, precisionscore))
print("Fold {0} accuracy_score: {1}".format(fold, accuracy))
print("Fold {0} F1_score: {1}".format(fold, F1score))
mean_precisionscore = np.mean(Precisionscore_list)
 mean_accuracy = np.mean(accuracyscore_list)
 mean_F1= np.mean(F1_score_list)
 print("Mean precision_score: {0}".format(mean_precisionscore))
print("Mean accuracy_score: {0}".format(mean_accuracy))
print("Mean F1_score: {0}".format(mean_F1))
return
def decay_4_first_10_then_1_f(array):
arr=[k for k in array]
for j in range(len(array)) :
if j<10:
arr[j]*=1./4.
 else :
arr[j]=0
return arr
	
def decay_16_first_10_then_1_f(array):
arr=[k for k in array]
for j in range(len(array)) :
if j<10:
arr[j]*=1./16.
 else :
arr[j]=0
return arr
def decay_2_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=1./2.
 return arr
def decay_4_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=1./4.
 return arr
	
def acceleratorx2_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=2.
 return arr
def decay_1_5_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=1./1.5
return arr
def decay_1_2_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=1./1.2
return arr
def decay_1_1_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=1./1.1
return arr
def stay_same_f(array):
arr=[0.0 for k in array]
return arr
def decay_2_last_12_linear_inter_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=1/2.
 arr12= arr[-12]
for j in range(0, 12):
arr[len(arr)-12 +j]= arr12/(j+1.)
return arr
def decay_1_5_last_12_linear_inter_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=1/(1.5)
 arr12= arr[-12]
for j in range(0, 12):
arr[len(arr)-12 +j]= arr12/(j+1.)
return arr
def decay_1_2_last_12_linear_inter_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=1./(1.2)
 arr12= arr[-12]
for j in range(0, 12):
arr[len(arr)-12 +j]= arr12/(j+1.)
return arr
def decay_4_last_12_linear_inter_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=1/4.
 arr12= arr[-12]
for j in range(0, 12):
arr[len(arr)-12 +j]= arr12/(j+1.)
return arr
def decay_8_last_12_linear_inter_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=1/8.
 arr12= arr[-12]
for j in range(0, 12):
arr[len(arr)-12 +j]= arr12/(j+1.)
return arr
def linear_last_12_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]=array[j]
arr12= arr[-12]
for j in range(0, 12):
arr[len(arr)-12 +j]= arr12/(j+1.)
return arr
decay_4_first_10_then_1 =[ "Jilin_China","Tianjin_China"]
decay_4_first_10_then_1_fatality=[]
decay_16_first_10_then_1 =["Beijing_China","Fujian_China","Jiangsu_China"]
decay_16_first_10_then_1_fatality=[]
decay_4=["nan_Central African Republic","Shanghai_China","nan_Equatorial Guinea","nan_Korea, South","nan_Zambia"]
decay_4_fatality=["Heilongjiang_China","Inner Mongolia_China"]
decay_2 =["nan_Burundi","Faroe Islands_Denmark","nan_Eritrea","nan_Gabon","nan_Italy","nan_Brunei""Heilongjiang_China","Inner Mongolia_China",
"Shanxi_China","nan_Congo(Kinshasa)","nan_Cyprus","Reunion_France","Sint Maarten_Netherlands","nan_Angola"]
decay_2_fatality=["nan_Iran"]
stay_same=["nan_Diamond Princess"]
stay_same_fatality=["Beijing_China","Fujian_China","Qinghai_China","Jiangsu_China","Jilin_China","Macau_China","Ningxia_China","Tianjin_China"
,"Faroe Islands_Denmark","Shanxi_China"]
normal=[]
normal_fatality=["nan_Andorra","New South Wales_Australia","nan_Korea, South","New York_US","nan_Austria","Alberta_Canada","Nova Scotia_Canada",
"Saskatchewan_Canada","nan_Costa Rica","nan_Croatia","nan_Finland","Martinique_France","Mayotte_France","nan_Hungary","nan_Iceland","nan_Latvia","nan_Czechia"
,"nan_Luxembourg","nan_Malta","nan_Montenegro","nan_New Zealand","nan_Norway","nan_South Africa","Hawaii_US","Ohio_US","Pennsylvania_US","nan_Uruguay","nan_Venezuela",
"nan_West Bank and Gaza"]
decay_4_last_12_linear_inter =[ "nan_Dominica","New Caledonia_France",
"Saint Barthelemy_France","nan_Gambia","nan_Grenada","nan_Holy See",
"nan_Mauritania","nan_Nicaragua","nan_Saint Lucia","nan_Saint Vincent and the Grenadines",
"nan_Seychelles","nan_Suriname","nan_Mongolia",
"Montserrat_United Kingdom","Turks and Caicos Islands_United Kingdom", "Hong Kong_China","Northern Territory_Australia","Northwest Territories_Canada","Yukon_Canada"
,"Guangdong_China","nan_MS Zaandam","nan_Maldives","nan_Saint Kitts and Nevis","nan_Sao Tome and Principe","nan_South Sudan","nan_Western Sahara"]
decay_4_last_12_linear_inter_fatality=["nan_Somalia"]
decay_2_last_12_linear_inter =[ "nan_Chad","French Polynesia_France","nan_Laos","nan_Nepal","nan_Sudan","nan_Tanzania","Bermuda_United Kingdom","Cayman Islands_United Kingdom"
,"nan_Cabo Verde","nan_Eswatini","nan_Guyana","nan_Liberia","nan_Malawi","nan_Mozambique","nan_Somalia","nan_Timor-Leste","Falkland Islands(Malvinas)_United Kingdom",
"nan_Zimbabwe"]
decay_2_last_12_linear_inter_fatality=[]
decay_1_5 =["nan_Cambodia","nan_Croatia","nan_Denmark",
"nan_Bahamas","nan_Bahrain","nan_Barbados" ,"Manitoba_Canada","New Brunswick_Canada",
"Saskatchewan_Canada","nan_France","nan_Libya","nan_Malta",
	"Aruba_Netherlands","nan_Niger","nan_Spain","Virgin Islands_US",
	"Channel Islands_United Kingdom","Gibraltar_United Kingdom","nan_United Kingdom",'nan_Burma',"nan_Congo(Brazzaville)",
	"French Guiana_France","Martinique_France","Mayotte_France","nan_Georgia","nan_Iran","nan_New Zealand","nan_Syria",
	"nan_West Bank and Gaza"]
decay_1_5_fatality=["nan_Cuba","nan_Morocco","Ohio_US","Pennsylvania_US","Puerto Rico_US" ,"nan_Uzbekistan","nan_Cyprus","nan_France","nan_Niger","South Dakota_US"]
acceleratorx2=[]
acceleratorx2_fatality=["Newfoundland and Labrador_Canada"]
decay_1_5_last_12_linear_inter =["nan_Antigua and Barbuda","nan_Botswana","nan_Haiti","nan_Madagascar"]
decay_1_5_last_12_linear_inter_fatality =[]
decay_1_2 =["nan_Albania","nan_Andorra","New South Wales_Australia","nan_Austria","nan_Azerbaijan","nan_Bangladesh","nan_Belize","Alberta_Canada","nan_Mauritius","nan_Monaco","nan_Montenegro",
"Newfoundland and Labrador_Canada","nan_Costa Rica","nan_Czechia","nan_Dominican Republic","nan_Estonia","Guadeloupe_France","nan_Iceland","nan_Latvia","nan_Liechtenstein","nan_Luxembourg","nan_Norway",
"nan_Poland","nan_Rwanda","nan_South Africa","nan_Trinidad and Tobago","Arizona_US","California_US","Colorado_US","Connecticut_US","Georgia_US","Hawaii_US","Maine_US","New York_US","Oklahoma_US",
"Oregon_US","Pennsylvania_US","Tennessee_US","Washington_US","nan_Uruguay","nan_Venezuela","nan_Vietnam"]
decay_1_2_fatality=["nan_Afghanistan","nan_Belarus","nan_Djibouti","nan_Mali","nan_Mexico","nan_Peru","nan_Serbia","Kentucky_US","Puerto Rico_US","Texas_US","nan_Ukraine"]
decay_1_1 =["nan_Algeria","nan_Argentina","Tasmania_Australia","nan_Bolivia","Nova Scotia_Canada","nan_Finland","nan_Hungary","nan_Iraq",
"nan_Jordan","nan_Kyrgyzstan","nan_Malaysia","Alabama_US","Iowa_US","Michigan_US","Missouri_US","Nebraska_US","North Dakota_US ","Ohio_US","South Carolina_US"]
decay_1_1_fatality=["nan_Afghanistan","nan_Ecuador","nan_Kenya","nan_Korea, South","nan_Kosovo","Indiana_US","New Mexico_US","Oregon_US","Rhode Island_US","Tennessee_US","nan_United Arab Emirates"]
decay_1_2_last_12_linear_inter =[]
decay_1_2_last_12_linear_inter_fatality=["nan_Angola" ]
linear_last_12=["nan_Afghanistan","nan_Ireland","nan_Benin"]
linear_last_12_fatality=["nan_Guatemala","nan_Ireland"]
decay_8_last_12_linear_inter =[ "nan_Bhutan","Prince Edward Island_Canada","Greenland_Denmark","nan_Fiji","Saint Pierre and Miquelon_France","St Martin_France","nan_Namibia",
"Bonaire, Sint Eustatius and Saba_Netherlands","Curacao_Netherlands","nan_Papua New Guinea","Anguilla_United Kingdom","British Virgin Islands_United Kingdom",]
decay_8_last_12_linear_inter_fatality=[]
tr_frame=train_frame
features_train=tr_frame[names].values
standard_confirmed_train=tr_frame["ConfirmedCases"].values
standard_fatalities_train=tr_frame["Fatalities"].values
current_confirmed_train=tr_frame["ConfirmedCases"].values
features_cv=[]
name_cv=[]
standard_confirmed_cv=[]
standard_fatalities_cv=[]
names_=tr_frame["key"].values
training_horizon=int(features_train.shape[0]/len(unique_keys))
print("training horizon = ",training_horizon)
 for dd in range(training_horizon-1,features_train.shape[0],training_horizon):
features_cv.append(features_train[dd])
 name_cv.append(names_[dd])
 standard_confirmed_cv.append(standard_confirmed_train[dd])
 standard_fatalities_cv.append(standard_fatalities_train[dd])
 print(name_cv[-1], standard_confirmed_cv[-1], standard_fatalities_cv[-1])
 features_cv=np.array(features_cv)
 preds_confirmed_cv=np.zeros(( features_cv.shape[0],horizon))
preds_confirmed_standard_cv=np.zeros(( features_cv.shape[0],horizon))
preds_fatalities_cv=np.zeros(( features_cv.shape[0],horizon))
preds_fatalities_standard_cv=np.zeros(( features_cv.shape[0],horizon))
overal_rmsle_metric_confirmed=0.0
for j in range(preds_confirmed_cv.shape[1]):
this_features_cv=features_cv
preds=predict(features_cv, input_name=model_directory +"confirmedc"+ str(j))
preds_confirmed_cv[:,j]=preds
print(" modelling confirmed, case %d, , original cv %d and after %d "%(j,this_features_cv.shape[0],this_features_cv.shape[0]))
predictions=[]
for ii in range(preds_confirmed_cv.shape[0]):
current_prediction=standard_confirmed_cv[ii]
if current_prediction==0 :
current_prediction=0.1
this_preds=preds_confirmed_cv[ii].tolist()
name=name_cv[ii]
reserve=this_preds[0]
	
if name in normal:
this_preds=this_preds	
	
elif name in decay_4_first_10_then_1:
this_preds=decay_4_first_10_then_1_f(this_preds)
 		
elif name in decay_16_first_10_then_1:
this_preds=decay_16_first_10_then_1_f(this_preds)		
		
elif name in decay_4_last_12_linear_inter:
this_preds=decay_4_last_12_linear_inter_f(this_preds)		
elif name in decay_8_last_12_linear_inter:
this_preds=decay_8_last_12_linear_inter_f(this_preds)		
elif name in decay_4:
this_preds=decay_4_f(this_preds)
 elif name in decay_2:
this_preds=decay_2_f(this_preds)
 elif name in decay_2_last_12_linear_inter:
this_preds=decay_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_5:
this_preds=decay_1_5_f(this_preds)
 elif name in decay_1_5_last_12_linear_inter:
this_preds=decay_1_5_last_12_linear_inter_f(this_preds)
 elif name in decay_1_2:
this_preds=decay_1_2_f(this_preds)
 elif name in decay_1_2_last_12_linear_inter:
this_preds=decay_1_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_1:
this_preds=decay_1_1_f(this_preds)
 elif name in linear_last_12:
this_preds=linear_last_12_f(this_preds)
 elif name in acceleratorx2:
this_preds=acceleratorx2_f(this_preds)
 elif name in stay_same or "China" in name:
this_preds=stay_same_f(this_preds)
 for j in range(preds_confirmed_cv.shape[1]):
current_prediction+=np.expm1(max(0,this_preds[j]))
preds_confirmed_standard_cv[ii][j]=current_prediction
for j in range(preds_confirmed_cv.shape[1]):
this_features_cv=features_cv
preds=predict(features_cv, input_name=model_directory +"fatalc"+ str(j))
preds_fatalities_cv[:,j]=preds
print(" modelling fatalities, case %d, original cv %d and after %d "%(j,this_features_cv.shape[0],this_features_cv.shape[0]))
predictions=[]
for ii in range(preds_fatalities_cv.shape[0]):
current_prediction=standard_fatalities_cv[ii]
this_preds=preds_fatalities_cv[ii].tolist()
name=name_cv[ii]
reserve=this_preds[0]
	
if name in normal_fatality:
this_preds=this_preds	 	
	
elif name in decay_4_first_10_then_1_fatality:
this_preds=decay_4_first_10_then_1_f(this_preds)
 		
elif name in decay_16_first_10_then_1_fatality:
this_preds=decay_16_first_10_then_1_f(this_preds)
 elif name in decay_4_last_12_linear_inter_fatality:
this_preds=decay_4_last_12_linear_inter_f(this_preds)		
elif name in decay_8_last_12_linear_inter_fatality:
this_preds=decay_8_last_12_linear_inter_f(this_preds)		
elif name in decay_4_fatality:
this_preds=decay_4_f(this_preds)		
		
elif name in decay_2_fatality:
this_preds=decay_2_f(this_preds)
 elif name in decay_2_last_12_linear_inter_fatality:
this_preds=decay_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_5_fatality:
this_preds=decay_1_5_f(this_preds)	
elif name in decay_1_5_last_12_linear_inter_fatality:
this_preds=decay_1_5_last_12_linear_inter_f(this_preds)
 elif name in decay_1_2_fatality:
this_preds=decay_1_2_f(this_preds)
 elif name in decay_1_2_last_12_linear_inter_fatality:
this_preds=decay_1_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_1_fatality:
this_preds=decay_1_1_f(this_preds)
 elif name in linear_last_12_fatality:
this_preds=linear_last_12_f(this_preds)
 elif name in acceleratorx2_fatality:
this_preds=acceleratorx2_f(this_preds)
 elif name in stay_same_fatality:
this_preds=stay_same_f(this_preds)
 elif name in normal:
this_preds=this_preds	
	
elif name in decay_4_first_10_then_1:
this_preds=decay_4_first_10_then_1_f(this_preds)
 		
elif name in decay_16_first_10_then_1:
this_preds=decay_16_first_10_then_1_f(this_preds)		
elif name in decay_4_last_12_linear_inter:
this_preds=decay_4_last_12_linear_inter_f(this_preds)		
elif name in decay_8_last_12_linear_inter:
this_preds=decay_8_last_12_linear_inter_f(this_preds)		
elif name in decay_4:
this_preds=decay_4_f(this_preds)
 		
elif name in decay_2:
this_preds=decay_2_f(this_preds)
 elif name in decay_2_last_12_linear_inter:
this_preds=decay_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_5:
this_preds=decay_1_5_f(this_preds)
 elif name in decay_1_5_last_12_linear_inter:
this_preds=decay_1_5_last_12_linear_inter_f(this_preds)
 elif name in decay_1_2:
this_preds=decay_1_2_f(this_preds)
 elif name in decay_1_2_last_12_linear_inter:
this_preds=decay_1_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_1:
this_preds=decay_1_1_f(this_preds)
 elif name in linear_last_12:
this_preds=linear_last_12_f(this_preds)
 elif name in acceleratorx2:
this_preds=acceleratorx2_f(this_preds)
 elif name in stay_same or "China" in name:
this_preds=stay_same_f(this_preds)
 for j in range(preds_fatalities_cv.shape[1]):
current_prediction+=np.expm1(max(0,this_preds[j]))
preds_fatalities_standard_cv[ii][j]=current_prediction<categorify>
classification_model=LogisticRegression(random_state=1)
 parameters = { 'C': [0.8,0.9,1.0,1.2],
'class_weight': [None, 'balanced'],
'solver': ['liblinear','sag', 'saga'],
}
best_model=hyper_parameter_tuning(classification_model,train_features_all,train_labels_all,parameters)
 features_to_use=recursiveFeature_elimination(df_train,best_model)
 df_filtered_features = df_train[features_to_use]
df_filtered_label =df_train.Zielvariable
run_kfold(best_model,df_filtered_features,df_filtered_label)
 sm_obj = SMOTE(random_state=43,sampling_strategy='auto')
 features_resampled, labels_resampled =sm_obj.fit_resample(df_filtered_features, df_filtered_label)
 best_model.fit(features_resampled,labels_resampled)
 df_test= test_data[features_to_use]
df_test= encode_categorical_features(df_test)
 if 'Dauer' in df_test.columns:
df_test[['Dauer']] = df_test[['Dauer']].apply(np.sqrt)
 scaler = MinMaxScaler(feature_range=(0, 1))
df_test[['Dauer']] = scaler.fit_transform(df_test[['Dauer']])
 test_features_all = df_test
test_labels_all =test_data.Zielvariable
predictions=best_model.predict(test_features_all)
 prediction_proba =best_model.predict_proba(test_features_all)
 pos_label_proba= prediction_proba[:,0]
key_to_confirmed_rate={}
key_to_fatality_rate={}
key_to_confirmed={}
key_to_fatality={}
print(len(features_cv), len(name_cv),len(standard_confirmed_cv),len(standard_fatalities_cv))
print(preds_confirmed_cv.shape,preds_confirmed_standard_cv.shape,preds_fatalities_cv.shape,preds_fatalities_standard_cv.shape)
 for j in range(len(name_cv)) :
key_to_confirmed_rate[name_cv[j]]=preds_confirmed_cv[j,:].tolist()
key_to_fatality_rate[name_cv[j]]=preds_fatalities_cv[j,:].tolist()
key_to_confirmed[name_cv[j]] =preds_confirmed_standard_cv[j,:].tolist()
key_to_fatality[name_cv[j]]=preds_fatalities_standard_cv[j,:].tolist() 
<merge>
y = predictions
pred = pos_label_proba
fpr, tpr, thresholds = metrics.roc_curve(y, pred, pos_label=0)
 AUC_score= metrics.auc(fpr, tpr)
 df_submission=pd.DataFrame()
df_submission['ID']=submission_data['ID']
df_submission['Expected']=pos_label_proba
df_submission.to_csv('Logistc_Regression_prediction.csv',index=False)
train_new=train[["Date","ConfirmedCases","Fatalities","key","rate_ConfirmedCases","rate_Fatalities"]]
test_new_count=pd.merge(test,train_new, how="left", left_on=["key","Date"], right_on=["key","Date"] ).reset_index(drop=True)
 test_new_count<categorify>
classification_model=GradientBoostingClassifier(random_state=1)
 parameters = { 'n_estimators': [100, 200, 300],
'max_features': ['sqrt','auto'],
'max_depth': [2, 3, 5],
}
best_model=hyper_parameter_tuning(classification_model,train_features_all,train_labels_all,parameters)
 features_to_use=recursiveFeature_elimination(df_train,best_model)
 df_filtered_features = df_train[features_to_use]
df_filtered_label =df_train.Zielvariable
run_kfold(best_model,df_filtered_features,df_filtered_label)
 sm_obj = SMOTE(random_state=43,sampling_strategy='auto')
 features_resampled, labels_resampled =sm_obj.fit_resample(df_filtered_features, df_filtered_label)
 best_model.fit(features_resampled,labels_resampled)
 df_test= test_data[features_to_use]
df_test= encode_categorical_features(df_test)
 if 'Dauer' in df_test.columns:
df_test[['Dauer']] = df_test[['Dauer']].apply(np.sqrt)
 scaler = MinMaxScaler(feature_range=(0, 1))
df_test[['Dauer']] = scaler.fit_transform(df_test[['Dauer']])
 test_features_all = df_test
test_labels_all =test_data.Zielvariable
predictions=best_model.predict(test_features_all)
 prediction_proba =best_model.predict_proba(test_features_all)
 pos_label_proba= prediction_proba[:,0]
def fillin_columns(frame,key_column, original_name, training_horizon, test_horizon, unique_values, key_to_values):
keys=frame[key_column].values
original_values=frame[original_name].values.tolist()
print(len(keys), len(original_values), training_horizon ,test_horizon,len(key_to_values))
for j in range(unique_values):
current_index=(j *(training_horizon +test_horizon)) +training_horizon
current_key=keys[current_index]
values=key_to_values[current_key]
co=0
for g in range(current_index, current_index + test_horizon):
original_values[g]=values[co]
co+=1
frame[original_name]=original_values
all_days=int(test_new.shape[0]/len(unique_keys))
tr_horizon=all_days-horizon
print(all_days,tr_horizon, horizon)
 fillin_columns(test_new_count,"key", 'ConfirmedCases', tr_horizon, horizon, len(unique_keys), key_to_confirmed)
 fillin_columns(test_new_count,"key", 'Fatalities', tr_horizon, horizon, len(unique_keys), key_to_fatality)
 fillin_columns(test_new_count,"key", 'rate_ConfirmedCases', tr_horizon, horizon, len(unique_keys), key_to_confirmed_rate)
 fillin_columns(test_new_count,"key", 'rate_Fatalities', tr_horizon, horizon, len(unique_keys), key_to_fatality_rate)
 test_dat=test_new['Fatalities'].values.tolist()
test_new_countfat=test_new_count['Fatalities'].values.tolist()
for j in range(len(test_dat)) :
if test_dat[j]<1:
test_dat[j]=test_new_countfat[j]
test_new2= test_new.copy()
test_new2['Fatalities']=test_dat
test_new2 
<save_to_csv>
y = predictions
pred = pos_label_proba
fpr, tpr, thresholds = metrics.roc_curve(y, pred, pos_label=0)
 AUC_score= metrics.auc(fpr, tpr)
 AUC_score
df_submission=pd.DataFrame()
df_submission['ID']=submission_data['ID']
df_submission['Expected']=pos_label_proba
df_submission.to_csv('Gradient_Boosting_prediction.csv',index=False)
submission=test_new2[["ForecastId","ConfirmedCases","Fatalities"]]
submission.to_csv("submission.csv", index=False)
<compute_test_metric>
classification_model=DecisionTreeClassifier(random_state=1)
 parameters = { 'class_weight': [None, 'balanced'],
'max_features': ['sqrt','auto','log2'],
'max_depth': [None, 2, 3, 5],
}
best_model=hyper_parameter_tuning(classification_model,train_features_all,train_labels_all,parameters)
 features_to_use=recursiveFeature_elimination(df_train,best_model)
 df_filtered_features = df_train[features_to_use]
df_filtered_label =df_train.Zielvariable
run_kfold(best_model,df_filtered_features,df_filtered_label)
 sm_obj = SMOTE(random_state=43,sampling_strategy='auto')
 features_resampled, labels_resampled =sm_obj.fit_resample(df_filtered_features, df_filtered_label)
 best_model.fit(features_resampled,labels_resampled)
 df_test= test_data[features_to_use]
df_test= encode_categorical_features(df_test)
 if 'Dauer' in df_test.columns:
df_test[['Dauer']] = df_test[['Dauer']].apply(np.sqrt)
 scaler = MinMaxScaler(feature_range=(0, 1))
df_test[['Dauer']] = scaler.fit_transform(df_test[['Dauer']])
 test_features_all = df_test
test_labels_all =test_data.Zielvariable
predictions=best_model.predict(test_features_all)
 prediction_proba =best_model.predict_proba(test_features_all)
 pos_label_proba= prediction_proba[:,0]
y = predictions
pred = pos_label_proba
fpr, tpr, thresholds = metrics.roc_curve(y, pred, pos_label=0)
 AUC_score= metrics.auc(fpr, tpr)
 df_submission=pd.DataFrame()
df_submission['ID']=submission_data['ID']
df_submission['Expected']=pos_label_proba
df_submission.to_csv('Decision_tree_prediction.csv',index=False)
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5
def fix_target(frame, key, target, new_target_name="target"):
corrections = 0
group_keys = frame[ key].values.tolist()
target = frame[target].values.tolist()
for i in range(1, len(group_keys)- 1):
previous_group = group_keys[i - 1]
current_group = group_keys[i]
previous_value = target[i - 1]
current_value = target[i]
if current_group == previous_group:
if current_value<previous_value:
current_value=previous_value
target[i] =current_value
target[i] =max(0,target[i])
 frame[new_target_name] = np.array(target)
 def rate(frame, key, target, new_target_name="rate"):
corrections = 0
group_keys = frame[ key].values.tolist()
target = frame[target].values.tolist()
rate=[1.0 for k in range(len(target)) ]
for i in range(1, len(group_keys)- 1):
previous_group = group_keys[i - 1]
current_group = group_keys[i]
previous_value = target[i - 1]
current_value = target[i]
if current_group == previous_group:
if previous_value!=0.0:
rate[i]=current_value/previous_value
rate[i] =max(1,rate[i])
 frame[new_target_name] = np.array(rate)
 def get_data_by_key(dataframe, key, key_value, fields=None):
mini_frame=dataframe[dataframe[key]==key_value]
if not fields is None:
mini_frame=mini_frame[fields].values
return mini_frame
directory="/kaggle/input/covid19-global-forecasting-week-4/"
model_directory="/kaggle/input/modelv3-dir/model"
geo_dir=None
extra_stable_columns=None
group_by_columns=None
if not group_by_columns is None and "continent" in group_by_columns:
assert not geo_dir is None
train=pd.read_csv(directory + "train.csv", parse_dates=["Date"] , engine="python")
 test=pd.read_csv(directory + "test.csv", parse_dates=["Date"], engine="python")
 train["key"]=train[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 test["key"]=test[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 if not geo_dir is None:
region_metadata=pd.read_csv(geo_dir+"region_metadata.csv")
 train=pd.merge(train,region_metadata, how="left", left_on=["Province_State","Country_Region"], right_on=["Province_State","Country_Region"])
 test=pd.merge(test,region_metadata, how="left", left_on=["Province_State","Country_Region"], right_on=["Province_State","Country_Region"])
 train.to_csv(directory + "train_plus_geo.csv", index=False)
 group_names=None
if not group_by_columns is None and len(group_by_columns)>0:
group_names=[]
for group in group_by_columns:
groupss=train[["Date", group,target1,target2]]
grp=groupss.groupby(["Date", group], as_index=False ).sum()
grp.columns=["Date", group,group +"_" + target1,group +"_" + target2 ]
train=pd.merge(train,grp, how="left", left_on=["Date", group,group], right_on=["Date", group,group])
 for gr in [group +"_" + target1,group +"_" + target2]:
rate(train, key, gr, new_target_name="rate_" +gr)
 group_names+=["rate_" +gr]
train.to_csv(directory + "train_plus_groups.csv", index=False)
 max_train_date=train["Date"].max()
max_test_date=test["Date"].max()
horizon=(max_test_date-max_train_date ).days
print("horizon", int(horizon))
print("max_train_date",(max_train_date))
print("max_test_date",(max_test_date))
target1="ConfirmedCases"
target2="Fatalities"
key="key" 
<categorify>
classification_model=RandomForestClassifier(random_state=1)
 parameters = { 'class_weight': [None, 'balanced'],
'n_estimators': [100, 200, 300],
'max_features': ['sqrt','auto'],
'max_depth': [None, 2, 3, 5],
}
best_model=hyper_parameter_tuning(classification_model,train_features_all,train_labels_all,parameters)
 features_to_use=recursiveFeature_elimination(df_train,best_model)
 df_filtered_features = df_train[features_to_use]
df_filtered_label =df_train.Zielvariable
run_kfold(best_model,df_filtered_features,df_filtered_label)
 sm_obj = SMOTE(random_state=43,sampling_strategy='auto')
 features_resampled, labels_resampled =sm_obj.fit_resample(df_filtered_features, df_filtered_label)
 best_model.fit(features_resampled,labels_resampled)
 df_test= test_data[features_to_use]
df_test= encode_categorical_features(df_test)
 if 'Dauer' in df_test.columns:
df_test[['Dauer']] = df_test[['Dauer']].apply(np.sqrt)
 scaler = MinMaxScaler(feature_range=(0, 1))
df_test[['Dauer']] = scaler.fit_transform(df_test[['Dauer']])
 test_features_all = df_test
test_labels_all =test_data.Zielvariable
predictions=best_model.predict(test_features_all)
 prediction_proba =best_model.predict_proba(test_features_all)
 pos_label_proba= prediction_proba[:,0]
y = predictions
pred = pos_label_proba
fpr, tpr, thresholds = metrics.roc_curve(y, pred, pos_label=0)
 AUC_score= metrics.auc(fpr, tpr)
 df_submission=pd.DataFrame()
df_submission['ID']=submission_data['ID']
df_submission['Expected']=pos_label_proba
df_submission.to_csv('Random_Forest_Classifier_prediction.csv',index=False)