Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<create_dataframe>
import numpy as np
import pandas as pd
test_y_conf = pd.DataFrame(test_y_conf )<rename_columns>
sns.set(style='white')
 %matplotlib inline
SEED=101
test_y_conf.rename(columns={0: 'ConfirmedCases'}, inplace=True )<concatenate>
def concat_df(train_data, test_data):
return pd.concat([train_data, test_data], sort=True ).reset_index(drop=True)
 def divide_df(all_data):
return all_data.loc[:890], all_data.loc[891:].drop(['Survived'], axis=1)
 def display_missing(df):
print(df.name)
 print('-'*77)
 for col in df.columns:
print('the number of missing values in {0}: {1}'.format(col, df[col].isnull().sum()))
sub1 = pd.concat([sub1, test_y_conf], axis=1 )<concatenate>
df_train = pd.read_csv('/kaggle/input/titanic/train.csv')
 df_test = pd.read_csv('/kaggle/input/titanic/test.csv')
 df_all = concat_df(df_train, df_test)
 df_train.name = 'Training Set'
df_test.name = 'Testing Set'
df_all.name = 'All Set'
dfs = [df_train, df_test]
drop_list = []
passengerId = df_test['PassengerId']
print('Number of training example: {}'.format(df_train.shape[0]))
print('Number of testing example: {}
'.format(df_test.shape[0]))
print('Training X shape: {}'.format(df_train.shape))
print('Training Y shape: {}
'.format(df_train['Survived'].shape[0]))
print('Testing X shape: {}'.format(df_test.shape))
print('Testing Y shape: {}
'.format(df_test.shape[0]))
print(df_train.columns)
 print(df_test.columns)
test = pd.concat([test, test_y_conf], axis=1 )<split>
for df in dfs:
display_missing(df )
<choose_model_class>
age_median_by_pclass_and_sex = df_all.groupby(['Sex','Pclass'] ).median() ['Age']
for i in df_all['Pclass'].unique() :
for j in df_all['Sex'].unique() :
print('The median age of class{0} and {1} is: {2}'.format(i,j,age_median_by_pclass_and_sex[j][i]))
df_all['Age'] = df_all.groupby(['Sex','Pclass'])['Age'].apply(lambda x: x.fillna(x.median()))
display_missing(df_all )
xgb1 = XGBRegressor(
 n_estimators = 500,
max_depth = 20,
learning_rate = 0.1,
)<train_model>
df_all['Embarked'] = df_all['Embarked'].fillna('S' )
xgb1.fit(X_train_with_confirmed, y_train_fatal )<predict_on_test>
df_all.loc[df_all['Fare'].isnull() ]
test_y_fatal = xgb1.predict(test)
<feature_engineering>
med = df_all.groupby(['Pclass','Parch','SibSp'])['Fare'].median() [3][0][0]
df_all['Fare'] = df_all['Fare'].fillna(med )
test_y_fatal = pd.DataFrame(test_y_fatal)
 test_y_fatal.rename(columns={0: 'Fatalities'}, inplace=True)
 test_y_fatal[test_y_fatal < 0] = 0<concatenate>
df_all['Deck'] = df_all['Cabin'].apply(lambda x: x[0] if pd.notnull(x)else 'M')
 df_all.groupby('Deck')['Survived'].mean()
sub1 = pd.concat([sub1, test_y_fatal], axis=1 )<save_to_csv>
df_all_decks = df_all.groupby(['Deck','Pclass'] ).count().drop(columns=['Survived',
'Sex','SibSp','Age','Parch','Fare','Embarked', 'Cabin', 'PassengerId', 'Ticket'] ).rename(columns={'Name':'Count'} ).transpose()
df_all_decks
sub1.to_csv("submission.csv" , index = False )<train_on_grid>
def get_pclass_dist(df):
deck_counts = {'A': {}, 'B': {}, 'C': {}, 'D': {}, 'E': {}, 'F': {}, 'G': {}, 'M': {}, 'T': {}}
decks = df.columns.levels[0]
for deck in decks:
for pclass in range(1, 4):
try:
count = df[deck][pclass][0]
deck_counts[deck][pclass] = count
except KeyError:
deck_counts[deck][pclass] = 0
df_decks = pd.DataFrame(deck_counts)
 deck_percentages = {}
for col in df_decks.columns:
deck_percentages[col] = [(count / df_decks[col].sum())* 100 for count in df_decks[col]]
return deck_counts, deck_percentages
all_deck_count, all_deck_per = get_pclass_dist(df_all_decks)
 print(all_deck_count)
 print(all_deck_per )
<split>
df_all['Deck'] = df_all['Deck'].replace('T','A')
 df_all['Deck'] = df_all['Deck'].replace(['A','B','C'],'ABC')
 df_all['Deck'] = df_all['Deck'].replace(['D','E'],'DE')
 df_all['Deck'] = df_all['Deck'].replace(['F','G'],'FG')
 df_all['Deck'].value_counts()
<compute_test_metric>
display_missing(df_all)
 drop_list.append('Cabin')
 drop_list.append('PassengerId' )
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5
def fix_target(frame, key, target, new_target_name="target"):
corrections = 0
group_keys = frame[ key].values.tolist()
target = frame[target].values.tolist()
for i in range(1, len(group_keys)- 1):
previous_group = group_keys[i - 1]
current_group = group_keys[i]
previous_value = target[i - 1]
current_value = target[i]
if current_group == previous_group:
if current_value<previous_value:
current_value=previous_value
target[i] =current_value
target[i] =max(0,target[i])
 frame[new_target_name] = np.array(target)
 def rate(frame, key, target, new_target_name="rate"):
corrections = 0
group_keys = frame[ key].values.tolist()
target = frame[target].values.tolist()
rate=[1.0 for k in range(len(target)) ]
for i in range(1, len(group_keys)) :
previous_group = group_keys[i - 1]
current_group = group_keys[i]
previous_value = target[i - 1]
current_value = target[i]
if current_group == previous_group:
if previous_value!=0.0:
rate[i]=current_value/previous_value
rate[i] =max(1,rate[i])
 frame[new_target_name] = np.array(rate)
 def difference(frame, key, target, new_target_name="diff"):
corrections = 0
group_keys = frame[ key].values.tolist()
target = frame[target].values.tolist()
rate=[0 for k in range(len(target)) ]
for i in range(1, len(group_keys)) :
previous_group = group_keys[i - 1]
current_group = group_keys[i]
previous_value = target[i - 1]
current_value = target[i]
if current_group == previous_group:
rate[i]=np.log1p(max(0,current_value-previous_value))
rate[i] =max(0,rate[i])
 frame[new_target_name] = np.array(rate)
 def get_data_by_key(dataframe, key, key_value, fields=None):
mini_frame=dataframe[dataframe[key]==key_value]
if not fields is None:
mini_frame=mini_frame[fields].values
return mini_frame
directory="/kaggle/input/covid19-global-forecasting-week-4/"
model_directory="/kaggle/input/model-dirv5/model"
us_county="/kaggle/input/covid19-us-county-jhu-data-demographics/covid_us_county"
geo_dir=None
extra_stable_columns=None
group_by_columns=None
if not group_by_columns is None and "continent" in group_by_columns:
assert not geo_dir is None
train=pd.read_csv(directory + "train.csv", parse_dates=["Date"] , engine="python")
 test=pd.read_csv(directory + "test.csv", parse_dates=["Date"], engine="python")
 train["key"]=train[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 test["key"]=test[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 if not geo_dir is None:
region_metadata=pd.read_csv(geo_dir+"region_metadata.csv")
 km=KMeans(n_clusters=30, init='k-means++', n_init=10, max_iter=300, tol=0.0001,
precompute_distances='auto', verbose=0, random_state=1234, copy_x=True, n_jobs=None, algorithm='auto')
 region_metadata["clusters"]=km.fit_predict(region_metadata[["lat","lon"]].values)
 train=pd.merge(train,region_metadata, how="left", left_on=["Province_State","Country_Region"], right_on=["Province_State","Country_Region"])
 test=pd.merge(test,region_metadata, how="left", left_on=["Province_State","Country_Region"], right_on=["Province_State","Country_Region"])
 group_names=None
if not group_by_columns is None and len(group_by_columns)>0:
group_names=[]
for group in group_by_columns:
groupss=train[["Date", group,target1,target2]]
grp=groupss.groupby(["Date", group], as_index=False ).sum()
grp.columns=["Date", group,group +"_" + target1,group +"_" + target2 ]
train=pd.merge(train,grp, how="left", left_on=["Date", group,group], right_on=["Date", group,group])
 for gr in [group +"_" + target1,group +"_" + target2]:
rate(train, key, gr, new_target_name="rate_" +gr)
 group_names+=["rate_" +gr]
train.to_csv(directory + "train_plus_groups.csv", index=False)
 max_train_date=train["Date"].max()
max_test_date=test["Date"].max()
horizon=(max_test_date-max_train_date ).days
print("horizon", int(horizon))
target1="ConfirmedCases"
target2="Fatalities"
key="key"
train_supplamanteary2=None
new_data_file2=[us_county]
if not new_data_file2 is None:
trains=pd.read_csv(directory + "train.csv", parse_dates=["Date"] , engine="python")
 tests=pd.read_csv(directory + "test.csv", parse_dates=["Date"], engine="python")
 trains["key"]=train[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 tests["key"]=test[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 train_supplamanteary2=None
for new_data_files in new_data_file2:
sumplementary_data=pd.read_csv(new_data_files +".csv" , parse_dates=["date"] , engine="python")
 sumplementary_data.columns=["fips","Province_State","Country_Region","lat","long","Date",
"ConfirmedCases","state_code","Fatalities"]
sumplementary_data["key"]=sumplementary_data[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 sumplementary_data=sumplementary_data[["Date","key","ConfirmedCases","Fatalities","Province_State","Country_Region"]]
sumplementary_data["Id"]=[k for k in range(999999999,sumplementary_data.shape[0]+999999999)]
sumplementary_data=sumplementary_data[sumplementary_data.Date>= '2020-02-20']
if train_supplamanteary2 is None:
train_supplamanteary2=pd.concat([trains.reset_index(drop=True), sumplementary_data.reset_index(drop=True)], axis=0 ).reset_index(drop=True)
 else :
train_supplamanteary2=pd.concat([train_supplamanteary2.reset_index(drop=True), sumplementary_data.reset_index(drop=True)], axis=0 ).reset_index(drop=True)
 print(train_supplamanteary2.head(5))
train_supplamanteary2 = train_supplamanteary2.sort_values(["Country_Region","key","Id"], ascending =(True, True, True))
fix_target(train_supplamanteary2, key, target1, new_target_name=target1)
 fix_target(train_supplamanteary2, key, target2, new_target_name=target2)
 rate(train_supplamanteary2, key, target1, new_target_name="rate_" +target1)
 rate(train_supplamanteary2, key, target2, new_target_name="rate_" +target2)
 difference(train_supplamanteary2, key, target1, new_target_name="diff_" +target1)
 difference(train_supplamanteary2, key, target2, new_target_name="diff_" +target2)
 unique_keys_sup2=train_supplamanteary2[key].unique()
print(len(unique_keys_sup2))
if not geo_dir is None:
region_metadata=pd.read_csv(geo_dir+"region_metadata.csv")
 km=KMeans(n_clusters=30, init='k-means++', n_init=10, max_iter=300, tol=0.0001,
precompute_distances='auto', verbose=0, random_state=1234, copy_x=True, n_jobs=None, algorithm='auto')
 region_metadata["clusters"]=km.fit_predict(region_metadata[["lat","lon"]].values)
 train_supplamanteary2=pd.merge(train_supplamanteary2,region_metadata, how="left", left_on=["Province_State","Country_Region"], right_on=["Province_State","Country_Region"])
 if not group_by_columns is None and len(group_by_columns)>0:
group_names=[]
for group in group_by_columns:
groupss=train_supplamanteary2[["Date", group,target1,target2]]
grp=groupss.groupby(["Date", group], as_index=False ).sum()
grp.columns=["Date", group,group +"_" + target1,group +"_" + target2 ]
train_supplamanteary2=pd.merge(train_supplamanteary2,grp, how="left", left_on=["Date", group,group], right_on=["Date", group,group])
 for gr in [group +"_" + target1,group +"_" + target2]:
rate(train_supplamanteary2, key, gr, new_target_name="rate_" +gr)
 group_names+=["rate_" +gr]<categorify>
df_all.drop(drop_list,axis=1,inplace=True)
 df_train, df_test = divide_df(df_all)
 dfs = [df_train,df_test]
def get_lags(rate_array, current_index, size=20):
lag_confirmed_rate=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
lag_confirmed_rate[j]=rate_array[current_index-j]
else :
break
return lag_confirmed_rate
def days_ago_thresold_hit(full_array, indx, thresold):
days_ago_confirmed_count_10=-1
if full_array[indx]>thresold:
for j in range(indx,-1,-1):
entered=False
if full_array[j]<=thresold:
days_ago_confirmed_count_10=abs(j-indx)
 entered=True
break
if entered==False:
days_ago_confirmed_count_10=100
return days_ago_confirmed_count_10
def ewma_vectorized(data, alpha):
sums=sum([(alpha**(k+1)) *data[k] for k in range(len(data)) ])
 counts=sum([(alpha**(k+1)) for k in range(len(data)) ])
 return sums/counts
def generate_ma_std_window(rate_array, current_index, size=20, window=3):
ma_rate_confirmed=[-1 for k in range(size)]
std_rate_confirmed=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
ma_rate_confirmed[j]=np.mean(rate_array[max(0,current_index-j-window+1):current_index-j+1])
 std_rate_confirmed[j]=np.std(rate_array[max(0,current_index-j-window+1):current_index-j+1])
 else :
break
return ma_rate_confirmed, std_rate_confirmed
def generate_ewma_window(rate_array, current_index, size=20, window=3, alpha=0.05):
ewma_rate_confirmed=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
ewma_rate_confirmed[j]=ewma_vectorized(rate_array[max(0,current_index-j-window+1):current_index-j+1, ], alpha)
 else :
break
return ewma_rate_confirmed
def get_target(rate_col, indx, horizon=33, average=3, use_hard_rule=False):
target_values=[-1 for k in range(horizon)]
cou=0
for j in range(indx+1, indx+1+horizon):
if j<len(rate_col):
if average==1:
target_values[cou]=rate_col[j]
else :
if use_hard_rule and j +average <=len(rate_col):
target_values[cou]=np.mean(rate_col[j:j +average])
 else :
target_values[cou]=np.mean(rate_col[j:min(len(rate_col),j +average)])
 cou+=1
else :
break
return target_values
def get_target_count(rate_col, indx, horizon=33, average=3, use_hard_rule=False):
target_values=[-1 for k in range(horizon)]
cou=0
for j in range(indx+1, indx+1+horizon):
if j<len(rate_col):
if average==1:
target_values[cou]=rate_col[j]
else :
if use_hard_rule and j +average <=len(rate_col):
target_values[cou]=np.mean(rate_col[j:j +average])
 else :
target_values[cou]=np.mean(rate_col[j:min(len(rate_col),j +average)])
 cou+=1
else :
break
return target_values
def dereive_features(frame, confirmed, fatalities, rate_confirmed, rate_fatalities, count_confirmed, count_fatalities ,
horizon ,size=20, windows=[3,7], days_back_confimed=[1,10,100], days_back_fatalities=[1,2,10],
extra_data=None, groups_data=None, windows_group=[3,7], size_group=20,
days_back_confimed_group=[1,10,100]):
targets=[]
if not extra_data is None:
assert len(extra_stable_columns)==extra_data.shape[1]
if not groups_data is None:
assert len(group_names)==groups_data.shape[1]
names=[]
for day in days_back_confimed:
names+=["days_ago_confirmed_count_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["ma" + str(window)+ "_count_confirmed" + str(k+1)for k in range(size)]
for day in days_back_fatalities:
names+=["days_ago_fatalitiescount_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["ma" + str(window)+ "_count_fatalities" + str(k+1)for k in range(size)]
names+=["confirmed_level"]
names+=["fatalities_level"]
if not extra_data is None:
names+=[k for k in extra_stable_columns]
if not groups_data is None:
for gg in range(groups_data.shape[1]):
for day in days_back_confimed_group:
names+=["days_ago_grooupcount_" + str(gg+1)+ "_" + str(day)]
for window in windows_group:
names+=["ma_group_" + str(gg+1)+ "_" + str(window)+ "_rate_" + str(k+1)for k in range(size_group)]
names+=["confirmed_plus" + str(k+1)for k in range(horizon)]
names+=["fatalities_plus" + str(k+1)for k in range(horizon)]
names+=["confirmed_count_plus" + str(k+1)for k in range(horizon)]
names+=["fatalities_count_plus" + str(k+1)for k in range(horizon)]
features=[]
for i in range(len(confirmed)) :
row_features=[]
lag_confirmed_rate=get_lags(rate_confirmed, i, size=size)
 for day in days_back_confimed:
days_ago_confirmed_count_10=days_ago_thresold_hit(confirmed, i, day)
 row_features+=[days_ago_confirmed_count_10]
for window in windows:
ma3_rate_confirmed,std3_rate_confirmed= generate_ma_std_window(rate_confirmed, i, size=size, window=window)
 row_features+= ma3_rate_confirmed
ma3_count_confirmed,std3_count_confirmed= generate_ma_std_window(count_confirmed, i, size=size, window=window)
 row_features+= ma3_count_confirmed
lag_fatalities_rate=get_lags(rate_fatalities, i, size=size)
 for day in days_back_fatalities:
days_ago_fatalitiescount_2=days_ago_thresold_hit(fatalities, i, day)
 row_features+=[days_ago_fatalitiescount_2]
for window in windows:
ma3_rate_fatalities,std3_rate_fatalities= generate_ma_std_window(rate_fatalities, i, size=size, window=window)
 row_features+= ma3_rate_fatalities
ma3_count_fatalities,std3_count_fatalities= generate_ma_std_window(count_fatalities, i, size=size, window=window)
 row_features+= ma3_count_fatalities
confirmed_level=0

 confirmed_level= confirmed[i]
row_features+=[confirmed_level]
fatalities_is_level=0

 fatalities_is_level= fatalities[i]
row_features+=[fatalities_is_level]
if not extra_data is None:
row_features+=extra_data[i].tolist()
if not groups_data is None:
for gg in range(groups_data.shape[1]):
this_group=groups_data[:,gg].tolist()
lag_group_rate=get_lags(this_group, i, size=size_group)
 for day in days_back_confimed_group:
days_ago_groupcount_2=days_ago_thresold_hit(this_group, i, day)
 row_features+=[days_ago_groupcount_2]
for window in windows_group:
ma3_rate_group,std3_rate_group= generate_ma_std_window(this_group, i, size=size_group, window=window)
 row_features+= ma3_rate_group
confirmed_plus=get_target(rate_confirmed, i, horizon=horizon)
 row_features+= confirmed_plus
fatalities_plus=get_target(rate_fatalities, i, horizon=horizon)
 row_features+= fatalities_plus
confirmed_plus=get_target(count_confirmed, i, horizon=horizon)
 row_features+= confirmed_plus
fatalities_plus=get_target(count_fatalities, i, horizon=horizon)
 row_features+= fatalities_plus
features.append(row_features)
 new_frame=pd.DataFrame(data=features, columns=names ).reset_index(drop=True)
 frame=frame.reset_index(drop=True)
 frame=pd.concat([frame, new_frame], axis=1)
 return frame
def feature_engineering_for_single_key(frame, group, key, horizon=33, size=20, windows=[3,7],
days_back_confimed=[1,10,100], days_back_fatalities=[1,2,10],
extra_stable_=None, group_nams=None,windows_group=[3,7],
size_group=20, days_back_confimed_group=[1,10,100]):
mini_frame=get_data_by_key(frame, group, key, fields=None)
 mini_frame_with_features=dereive_features(mini_frame, mini_frame["ConfirmedCases"].values,
mini_frame["Fatalities"].values, mini_frame["rate_ConfirmedCases"].values,
mini_frame["rate_Fatalities"].values, mini_frame["diff_ConfirmedCases"].values,
mini_frame["diff_Fatalities"].values,
horizon ,size=size, windows=windows,
days_back_confimed=days_back_confimed, days_back_fatalities=days_back_fatalities,
extra_data=mini_frame[extra_stable_].values if not extra_stable_ is None else None,
groups_data=mini_frame[group_nams].values if not group_nams is None else None,
windows_group=windows_group, size_group=size_group,
days_back_confimed_group=days_back_confimed_group)
 return mini_frame_with_features<define_search_space>
drop_list=[]
train_frame=[]
size=10
windows=[3]
days_back_confimed=[1,5,10,20,50,100,250,500,1000]
days_back_fatalities=[1,2,5,10,20,50]
size_group=10
windows_group=[3]
days_back_confimed_group=[1,10,100]
for unique_k in tqdm(unique_keys):
mini_frame=feature_engineering_for_single_key(train, key, unique_k, horizon=horizon, size=size,
windows=windows, days_back_confimed=days_back_confimed,
days_back_fatalities=days_back_fatalities,
extra_stable_=extra_stable_columns if extra_stable_columns is not None and len(extra_stable_columns)>0 else None,
group_nams=group_names,windows_group=windows_group,
size_group=size_group, days_back_confimed_group=days_back_confimed_group
).reset_index(drop=True)
 train_frame.append(mini_frame)
 train_frame = pd.concat(train_frame, axis=0 ).reset_index(drop=True)
 new_unique_keys=train_frame['key'].unique()
for kee in new_unique_keys:
if kee not in unique_keys:
print(kee , " is not there ")
 print("==================================================")
 train_frame_supplamanteary2=None
if not train_supplamanteary2 is None:
train_frame_supplamanteary2=[]
for unique_k in tqdm(unique_keys_sup2):
mini_frame_sup=feature_engineering_for_single_key(train_supplamanteary2, key, unique_k, horizon=horizon, size=size,
windows=windows, days_back_confimed=days_back_confimed,
days_back_fatalities=days_back_fatalities,
extra_stable_=extra_stable_columns if extra_stable_columns is not None and len(extra_stable_columns)>0 else None,
group_nams=group_names,windows_group=windows_group,
size_group=size_group, days_back_confimed_group=days_back_confimed_group ).reset_index(drop=True)
 train_frame_supplamanteary2.append(mini_frame_sup)
 train_frame_supplamanteary2 = pd.concat(train_frame_supplamanteary2, axis=0 ).reset_index(drop=True)
 new_unique_keys_sup2=train_frame_supplamanteary2['key'].unique() 
<train_model>
def corr_sort(df_train):
df_train_corr = df_train.corr().abs().unstack().sort_values(ascending=False ).reset_index()
df_train_corr.rename(columns={'level_0':'feature1','level_1':'feature2',0:'correlation'},inplace=True)
 df_train_corr = df_train_corr.iloc[::2]
return df_train_corr[(df_train_corr['correlation']>0.1)&(df_train_corr['correlation'] != 1)]
def bagged_set_train(X_ts,y_cs,wts, seed, estimators,xtest, xt=None,yt=None, output_name=None):
baggedpred=np.array([ 0.0 for d in range(0, xtest.shape[0])])
 for n in range(0, estimators):
params = {'objective': 'rmse',
'metric': 'rmse',
'boosting': 'gbdt',
'learning_rate': 0.005,
'drop_rate':0.01,
'skip_drop':0.6,
'uniform_drop':True,
'verbose': -1,
'num_leaves': 40,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'bagging_seed': seed + n,
'feature_fraction': 0.8,
'feature_fraction_seed': seed + n,
'min_data_in_leaf': 10,
'max_bin': 100,
'max_depth':20,
'reg_alpha':1,
'lambda_l2': 10,
'num_threads':6
}
d_train = lgb.Dataset(X_ts,y_cs, weight=wts, free_raw_data=False)
 if not type(yt)is type(None):
d_cv = lgb.Dataset(xt,yt, free_raw_data=False, reference=d_train)
 model = lgb.train(params,d_train,num_boost_round=500,
valid_sets=d_cv,
verbose_eval=50)
 else :
model = lgb.train(params,d_train,num_boost_round=500)
 preds=model.predict(xtest)
 baggedpred+=preds
if not output_name is None:
joblib.dump(( model), output_name)

baggedpred/= estimators
return baggedpred 
<predict_on_test>
corr_train = corr_sort(df_train)
 corr_test = corr_sort(df_test)
 print(corr_train)
 print(corr_test)
def predict(xtest,input_name=None):
baggedpred=np.array([ 0.0 for d in range(0, xtest.shape[0])])
 model= joblib.load(input_name)
 preds=model.predict(xtest)
 baggedpred+=preds
return baggedpred<define_variables>
display_missing(df_all )
names=[]
for day in days_back_confimed:
names+=["days_ago_confirmed_count_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
for day in days_back_fatalities:
names+=["days_ago_fatalitiescount_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["confirmed_level"]
names+=["fatalities_level"]
if not extra_stable_columns is None and len(extra_stable_columns)>0:
names+=[k for k in extra_stable_columns]
if not group_names is None:
for gg in range(len(group_names)) :
for day in days_back_confimed_group:
names+=["days_ago_grooupcount_" + str(gg+1)+ "_" + str(day)]
for window in windows_group:
names+=["ma_group_" + str(gg+1)+ "_" + str(window)+ "_rate_" + str(k+1)for k in range(size_group)] 
<define_variables>
drop_list.append('Fare')
 drop_list.append('Farec' )
<categorify>
drop_list.append('Age')
 drop_list.append('Agec' )
def decay_4_first_10_then_1_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
if j<10:
arr[j]=1.+(max(1,array[j])-1.) /4.
 else :
arr[j]=1.
 return arr
	
def decay_16_first_10_then_1_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
if j<10:
arr[j]=1.+(max(1,array[j])-1.) /16.
 else :
arr[j]=1.
 return arr	
def decay_2_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /2.
 return arr
def decay_4_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /4.
 return arr 	
	
def acceleratorx2_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) *2.
 return arr
def decay_1_5_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /1.5
return arr
def decay_1_2_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /1.2
return arr
def decay_1_1_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /1.1
return arr
def stay_same_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.
 return arr
def decay_2_last_12_linear_inter_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /2.
 arr12=(max(1,arr[-12])-1.) /12.
 for j in range(0, 12):
arr[len(arr)-12 +j]= max(1, 1 +(( arr12*12)-(j+1)*arr12))
return arr
def decay_1_2_last_12_linear_inter_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /1.2
arr12=(max(1,arr[-12])-1.) /12.
 for j in range(0, 12):
arr[len(arr)-12 +j]= max(1, 1 +(( arr12*12)-(j+1)*arr12))
return arr
def decay_1_5_last_12_linear_inter_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /1.5
arr12=(max(1,arr[-12])-1.) /12.
 for j in range(0, 12):
arr[len(arr)-12 +j]= max(1, 1 +(( arr12*12)-(j+1)*arr12))
return arr
def decay_4_last_12_linear_inter_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /4.
 arr12=(max(1,arr[-12])-1.) /12.
 for j in range(0, 12):
arr[len(arr)-12 +j]= max(1, 1 +(( arr12*12)-(j+1)*arr12))
return arr
def decay_8_last_12_linear_inter_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /8.
 arr12=(max(1,arr[-12])-1.) /12.
 for j in range(0, 12):
arr[len(arr)-12 +j]= max(1, 1 +(( arr12*12)-(j+1)*arr12))
return arr
def linear_last_12_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=max(1,array[j])
 arr12=(max(1,arr[-12])-1.) /12.
 for j in range(0, 12):
arr[len(arr)-12 +j]= max(1, 1 +(( arr12*12)-(j+1)*arr12))
return arr
decay_4_first_10_then_1 =[ "Jilin_China","Tianjin_China"]
decay_4_first_10_then_1_fatality=[]
decay_16_first_10_then_1 =["Beijing_China","Fujian_China","Jiangsu_China"]
decay_16_first_10_then_1_fatality=[]
decay_4=["nan_Central African Republic","Shanghai_China","nan_Korea, South","nan_Zambia"]
decay_4_fatality=["Heilongjiang_China","Inner Mongolia_China"]
decay_2 =["nan_Burundi","Faroe Islands_Denmark","nan_Eritrea","nan_Italy","nan_Brunei""Heilongjiang_China","Inner Mongolia_China",
"Shanxi_China","nan_Congo(Kinshasa)","nan_Cyprus","Reunion_France","Sint Maarten_Netherlands","nan_Angola"]
decay_2_fatality=["nan_Iran"]
stay_same=["nan_Diamond Princess"]
stay_same_fatality=["Beijing_China","Fujian_China","Qinghai_China","Jiangsu_China","Jilin_China","Macau_China","Ningxia_China","Tianjin_China"
,"Faroe Islands_Denmark","Shanxi_China"]
normal=[]
normal_fatality=["nan_Andorra","New South Wales_Australia","nan_Korea, South","New York_US","nan_Austria","Alberta_Canada","Nova Scotia_Canada",
"Saskatchewan_Canada","nan_Costa Rica","nan_Croatia","nan_Finland","Mayotte_France","nan_Hungary","nan_Iceland","nan_Latvia","nan_Czechia"
,"nan_Luxembourg","nan_Malta","nan_Montenegro","nan_New Zealand","nan_Norway","nan_South Africa","Hawaii_US","Ohio_US","Pennsylvania_US","nan_Uruguay","nan_Venezuela",
"nan_West Bank and Gaza"]
decay_4_last_12_linear_inter =[ "nan_Dominica","New Caledonia_France",
"Saint Barthelemy_France","nan_Gambia","nan_Grenada","nan_Holy See",
"nan_Mauritania","nan_Nicaragua","nan_Saint Lucia","nan_Saint Vincent and the Grenadines",
"nan_Seychelles","nan_Suriname","nan_Mongolia","nan_Cabo Verde",
"Montserrat_United Kingdom","Turks and Caicos Islands_United Kingdom", "Hong Kong_China","Northern Territory_Australia","Northwest Territories_Canada","Yukon_Canada",
"Guangdong_China","nan_MS Zaandam","nan_Maldives","nan_Saint Kitts and Nevis","nan_Sao Tome and Principe","nan_South Sudan","nan_Western Sahara"]
decay_4_last_12_linear_inter_fatality=["nan_Somalia"]
decay_2_last_12_linear_inter =[ "nan_Chad","French Polynesia_France","nan_Laos","nan_Nepal","nan_Sudan","Bermuda_United Kingdom","Cayman Islands_United Kingdom"
,"nan_Eswatini","nan_Guyana","nan_Liberia","nan_Malawi","nan_Mozambique","nan_Timor-Leste",
"Falkland Islands(Malvinas)_United Kingdom","nan_Equatorial Guinea"]
decay_2_last_12_linear_inter_fatality=[]
decay_1_5 =["nan_Cambodia","nan_Croatia","nan_Denmark",
"nan_Bahamas","nan_Bahrain","nan_Barbados" ,"Manitoba_Canada","New Brunswick_Canada",
"Saskatchewan_Canada","nan_France","nan_Libya","nan_Malta",
	"Aruba_Netherlands","nan_Niger","nan_Spain","Virgin Islands_US",
	"Channel Islands_United Kingdom","Gibraltar_United Kingdom","nan_United Kingdom",'nan_Burma',"nan_Congo(Brazzaville)",
	"French Guiana_France","Martinique_France","Mayotte_France","nan_Georgia","nan_Iran","nan_New Zealand",
	"nan_West Bank and Gaza"]
decay_1_5_fatality=["nan_Cuba","nan_Morocco","Ohio_US","Pennsylvania_US","Puerto Rico_US" ,"nan_Uzbekistan",
"nan_Cyprus","nan_France","nan_Niger","South Dakota_US","nan_Gabon"]
acceleratorx2=[]
acceleratorx2_fatality=["Newfoundland and Labrador_Canada"]
decay_1_5_last_12_linear_inter =["nan_Antigua and Barbuda","nan_Botswana","nan_Haiti","nan_Madagascar","nan_Somalia",
"nan_Zimbabwe"]
decay_1_5_last_12_linear_inter_fatality =["nan_Burma","nan_Cabo Verde","nan_Djibouti"]
decay_1_2 =["nan_Albania","nan_Andorra","New South Wales_Australia","nan_Austria","nan_Azerbaijan","nan_Bangladesh","nan_Belize","Alberta_Canada","nan_Mauritius","nan_Monaco","nan_Montenegro",
"Newfoundland and Labrador_Canada","nan_Costa Rica","nan_Czechia","nan_Dominican Republic","nan_Estonia","Guadeloupe_France","nan_Iceland","nan_Latvia","nan_Liechtenstein","nan_Luxembourg","nan_Norway",
"nan_Poland","nan_Rwanda","nan_South Africa","nan_Trinidad and Tobago","Arizona_US","California_US","Colorado_US","Connecticut_US","Georgia_US","Hawaii_US","Maine_US","New York_US","Oklahoma_US",
"Oregon_US","Pennsylvania_US","Tennessee_US","Washington_US","nan_Uruguay","nan_Venezuela","nan_Vietnam",
"nan_Gabon","nan_Syria","nan_Djibouti"]
decay_1_2_fatality=["nan_Afghanistan","nan_Belarus","nan_Mali","nan_Mexico","nan_Peru","nan_Serbia","Kentucky_US","Puerto Rico_US","Texas_US","nan_Ukraine"]
decay_1_1 =["nan_Algeria","nan_Argentina","Tasmania_Australia","Nova Scotia_Canada","nan_Finland","nan_Hungary","nan_Iraq",
"nan_Jordan","nan_Kyrgyzstan","nan_Malaysia","Alabama_US","Iowa_US","Michigan_US","Missouri_US","Nebraska_US","North Dakota_US ","Ohio_US","South Carolina_US"]
decay_1_1_fatality=["nan_Ecuador","nan_Kenya","nan_Korea, South","nan_Kosovo","Indiana_US","New Mexico_US","Oregon_US","Rhode Island_US","Tennessee_US","nan_United Arab Emirates"]
decay_1_2_last_12_linear_inter =[]
decay_1_2_last_12_linear_inter_fatality=["nan_Angola" ]
linear_last_12=["nan_Afghanistan","nan_Ireland","nan_Benin","nan_Tanzania"]
linear_last_12_fatality=["nan_Guatemala","nan_Ireland"]
decay_8_last_12_linear_inter =[ "nan_Bhutan","Prince Edward Island_Canada","Greenland_Denmark","nan_Fiji","Saint Pierre and Miquelon_France","St Martin_France","nan_Namibia",
"Bonaire, Sint Eustatius and Saba_Netherlands","Curacao_Netherlands","nan_Papua New Guinea","Anguilla_United Kingdom","British Virgin Islands_United Kingdom",]
decay_8_last_12_linear_inter_fatality=[]
tr_frame=train_frame
features_train=tr_frame[names].values
standard_confirmed_train=tr_frame["ConfirmedCases"].values
standard_fatalities_train=tr_frame["Fatalities"].values
current_confirmed_train=tr_frame["ConfirmedCases"].values
features_cv=[]
name_cv=[]
standard_confirmed_cv=[]
standard_fatalities_cv=[]
names_=tr_frame["key"].values
training_horizon=int(features_train.shape[0]/len(unique_keys))
print("training horizon = ",training_horizon)
 for dd in range(training_horizon-1,features_train.shape[0],training_horizon):
features_cv.append(features_train[dd])
 name_cv.append(names_[dd])
 standard_confirmed_cv.append(standard_confirmed_train[dd])
 standard_fatalities_cv.append(standard_fatalities_train[dd])
 print(name_cv[-1], standard_confirmed_cv[-1], standard_fatalities_cv[-1])
 features_cv=np.array(features_cv)
 preds_confirmed_cv=np.zeros(( features_cv.shape[0],horizon))
preds_confirmed_standard_cv=np.zeros(( features_cv.shape[0],horizon))
preds_fatalities_cv=np.zeros(( features_cv.shape[0],horizon))
preds_fatalities_standard_cv=np.zeros(( features_cv.shape[0],horizon))
overal_rmsle_metric_confirmed=0.0
for j in range(preds_confirmed_cv.shape[1]):
this_features_cv=features_cv
preds=predict(features_cv, input_name=model_directory +"confirmed"+ str(j))
preds_confirmed_cv[:,j]=preds
print(" modelling confirmed, case %d, , original cv %d and after %d "%(j,this_features_cv.shape[0],this_features_cv.shape[0]))
predictions=[]
for ii in range(preds_confirmed_cv.shape[0]):
current_prediction=standard_confirmed_cv[ii]
if current_prediction==0 :
current_prediction=0.1
this_preds=preds_confirmed_cv[ii].tolist()
name=name_cv[ii]
reserve=this_preds[0]
if name in normal:
this_preds=this_preds	
	
elif name in decay_4_first_10_then_1:
this_preds=decay_4_first_10_then_1_f(this_preds)
 		
elif name in decay_16_first_10_then_1:
this_preds=decay_16_first_10_then_1_f(this_preds)		
		
elif name in decay_4_last_12_linear_inter:
this_preds=decay_4_last_12_linear_inter_f(this_preds)		
elif name in decay_8_last_12_linear_inter:
this_preds=decay_8_last_12_linear_inter_f(this_preds)		
elif name in decay_4:
this_preds=decay_4_f(this_preds)
 elif name in decay_2:
this_preds=decay_2_f(this_preds)
 elif name in decay_2_last_12_linear_inter:
this_preds=decay_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_5:
this_preds=decay_1_5_f(this_preds)
 elif name in decay_1_5_last_12_linear_inter:
this_preds=decay_1_5_last_12_linear_inter_f(this_preds)
 elif name in decay_1_2:
this_preds=decay_1_2_f(this_preds)
 elif name in decay_1_2_last_12_linear_inter:
this_preds=decay_1_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_1:
this_preds=decay_1_1_f(this_preds)
 elif name in linear_last_12:
this_preds=linear_last_12_f(this_preds)
 elif name in acceleratorx2:
this_preds=acceleratorx2_f(this_preds)
 elif name in stay_same or "China" in name:
this_preds=stay_same_f(this_preds)
 for j in range(preds_confirmed_cv.shape[1]):
if j==0 and "nan_Congo(Brazzaville)" in name:
current_prediction=117.
 if j==0 and "nan_Cabo Verde" in name:
current_prediction=56.
 current_prediction*=max(1,this_preds[j])
 preds_confirmed_standard_cv[ii][j]=current_prediction
for j in range(preds_confirmed_cv.shape[1]):
this_features_cv=features_cv
preds=predict(features_cv, input_name=model_directory +"fatal"+ str(j))
preds_fatalities_cv[:,j]=preds
print(" modelling fatalities, case %d, original cv %d and after %d "%(j,this_features_cv.shape[0],this_features_cv.shape[0]))
predictions=[]
for ii in range(preds_fatalities_cv.shape[0]):
current_prediction=standard_fatalities_cv[ii]
this_preds=preds_fatalities_cv[ii].tolist()
name=name_cv[ii]
reserve=this_preds[0]
if name in normal_fatality:
this_preds=this_preds	 	
	
elif name in decay_4_first_10_then_1_fatality:
this_preds=decay_4_first_10_then_1_f(this_preds)
 		
elif name in decay_16_first_10_then_1_fatality:
this_preds=decay_16_first_10_then_1_f(this_preds)
 elif name in decay_4_last_12_linear_inter_fatality:
this_preds=decay_4_last_12_linear_inter_f(this_preds)		
elif name in decay_8_last_12_linear_inter_fatality:
this_preds=decay_8_last_12_linear_inter_f(this_preds)		
elif name in decay_4_fatality:
this_preds=decay_4_f(this_preds)		
		
elif name in decay_2_fatality:
this_preds=decay_2_f(this_preds)
 elif name in decay_2_last_12_linear_inter_fatality:
this_preds=decay_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_5_fatality:
this_preds=decay_1_5_f(this_preds)	
elif name in decay_1_5_last_12_linear_inter_fatality:
this_preds=decay_1_5_last_12_linear_inter_f(this_preds)
 elif name in decay_1_2_fatality:
this_preds=decay_1_2_f(this_preds)
 elif name in decay_1_2_last_12_linear_inter_fatality:
this_preds=decay_1_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_1_fatality:
this_preds=decay_1_1_f(this_preds)
 elif name in linear_last_12_fatality:
this_preds=linear_last_12_f(this_preds)
 elif name in acceleratorx2_fatality:
this_preds=acceleratorx2_f(this_preds)
 elif name in stay_same_fatality:
this_preds=stay_same_f(this_preds)
 elif name in normal:
this_preds=this_preds	
	
elif name in decay_4_first_10_then_1:
this_preds=decay_4_first_10_then_1_f(this_preds)
 		
elif name in decay_16_first_10_then_1:
this_preds=decay_16_first_10_then_1_f(this_preds)		
elif name in decay_4_last_12_linear_inter:
this_preds=decay_4_last_12_linear_inter_f(this_preds)		
elif name in decay_8_last_12_linear_inter:
this_preds=decay_8_last_12_linear_inter_f(this_preds)		
elif name in decay_4:
this_preds=decay_4_f(this_preds)
 		
elif name in decay_2:
this_preds=decay_2_f(this_preds)
 elif name in decay_2_last_12_linear_inter:
this_preds=decay_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_5:
this_preds=decay_1_5_f(this_preds)
 elif name in decay_1_5_last_12_linear_inter:
this_preds=decay_1_5_last_12_linear_inter_f(this_preds)
 elif name in decay_1_2:
this_preds=decay_1_2_f(this_preds)
 elif name in decay_1_2_last_12_linear_inter:
this_preds=decay_1_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_1:
this_preds=decay_1_1_f(this_preds)
 elif name in linear_last_12:
this_preds=linear_last_12_f(this_preds)
 elif name in acceleratorx2:
this_preds=acceleratorx2_f(this_preds)
 elif name in stay_same or "China" in name:
this_preds=stay_same_f(this_preds)
 for j in range(preds_fatalities_cv.shape[1]):
if current_prediction==0 and preds_confirmed_standard_cv[ii][j]>400 and(name not in stay_same and name not in stay_same_fatality and "China" not in name):
current_prediction=1.
 if j==0 and "Aruba_Netherlands" in name:
current_prediction=1.
 if j==0 and "nan_Monaco" in name:
current_prediction=3.
 if j==0 and 'nan_Costa Rica' in name or "Isle of Man_United Kingdom" in name:
current_prediction=4.
 if j==0 and "nan_Somalia" in name:
current_prediction=5
if j==0 and "Martinique_France" in name:
current_prediction=8.
 current_prediction*=max(1,this_preds[j])
 preds_fatalities_standard_cv[ii][j]=current_prediction 
<define_variables>
family_map = {1: 'Alone', 2: 'Small', 3: 'Small', 4: 'Small', 5: 'Medium', 6: 'Medium', 7: 'Large', 8: 'Large', 11: 'Large'}
df_all['Family'] = df_all['FamilySize'].map(family_map)
 df_all['Family']
key_to_confirmed_rate={}
key_to_fatality_rate={}
key_to_confirmed={}
key_to_fatality={}
print(len(features_cv), len(name_cv),len(standard_confirmed_cv),len(standard_fatalities_cv))
print(preds_confirmed_cv.shape,preds_confirmed_standard_cv.shape,preds_fatalities_cv.shape,preds_fatalities_standard_cv.shape)
 for j in range(len(name_cv)) :
key_to_confirmed_rate[name_cv[j]]=preds_confirmed_cv[j,:].tolist()
key_to_fatality_rate[name_cv[j]]=preds_fatalities_cv[j,:].tolist()
key_to_confirmed[name_cv[j]] =preds_confirmed_standard_cv[j,:].tolist()
key_to_fatality[name_cv[j]]=preds_fatalities_standard_cv[j,:].tolist() 
<merge>
drop_list.append('FamilySize')
 drop_list.append('Parch')
 drop_list.append('SibSp' )
train_new=train[["Date","ConfirmedCases","Fatalities","key","rate_ConfirmedCases","rate_Fatalities"]]
test_new=pd.merge(test,train_new, how="left", left_on=["key","Date"], right_on=["key","Date"] ).reset_index(drop=True)
 test_new<categorify>
def title_replace(row):
if row.Title == 'Mlle':
return 'Miss'
elif row.Title == 'Ms':
return 'Miss'
elif row.Title == 'Mme':
return 'Mrs'
elif row.Title in ['Lady', 'Countess','Capt', 'Col','Don', 'Dr', 'Major', 'Rev', 'Sir', 'Jonkheer', 'Dona']:
return 'Rare'
else:
return row.Title
def fillin_columns(frame,key_column, original_name, training_horizon, test_horizon, unique_values, key_to_values):
keys=frame[key_column].values
original_values=frame[original_name].values.tolist()
print(len(keys), len(original_values), training_horizon ,test_horizon,len(key_to_values))
for j in range(unique_values):
current_index=(j *(training_horizon +test_horizon)) +training_horizon
current_key=keys[current_index]
values=key_to_values[current_key]
co=0
for g in range(current_index, current_index + test_horizon):
original_values[g]=values[co]
co+=1
frame[original_name]=original_values
all_days=int(test_new.shape[0]/len(unique_keys))
tr_horizon=all_days-horizon
print(all_days,tr_horizon, horizon)
 fillin_columns(test_new,"key", 'ConfirmedCases', tr_horizon, horizon, len(unique_keys), key_to_confirmed)
 fillin_columns(test_new,"key", 'Fatalities', tr_horizon, horizon, len(unique_keys), key_to_fatality)
 fillin_columns(test_new,"key", 'rate_ConfirmedCases', tr_horizon, horizon, len(unique_keys), key_to_confirmed_rate)
 fillin_columns(test_new,"key", 'rate_Fatalities', tr_horizon, horizon, len(unique_keys), key_to_fatality_rate )<categorify>
df_all['Title'] = df_all['Name'].str.split(',',expand=True)[1].str.split('.',expand=True)[0].str.strip()
df_all['Title'] = df_all['Title'].replace(['Miss', 'Mrs','Ms', 'Mlle', 'Lady', 'Mme', 'the Countess', 'Dona'], 'Miss/Mrs/Ms')
 df_all['Title'] = df_all['Title'].replace(['Dr', 'Col', 'Major', 'Jonkheer', 'Capt', 'Sir', 'Don', 'Rev'], 'Dr/Military/Noble/Clergy')
<train_model>
df_all['Is_Married'] = 0
df_all['Is_Married'].loc[df_all['Title'] == 'Mrs'] = 1
def bagged_set_trainc(X_ts,y_cs,wts, seed, estimators,xtest, xt=None,yt=None, output_name=None):
baggedpred=np.array([ 0.0 for d in range(0, xtest.shape[0])])
 for n in range(0, estimators):
params = {'objective': 'rmse',
'metric': 'rmse',
'boosting': 'gbdt',
'learning_rate': 0.005,
'drop_rate':0.01,
'skip_drop':0.6,
'uniform_drop':True,
'verbose': -1,
'num_leaves': 30,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'bagging_seed': seed + n,
'feature_fraction': 0.8,
'feature_fraction_seed': seed + n,
'min_data_in_leaf': 10,
'max_bin': 100,
'max_depth':20,
'reg_alpha':1,
'lambda_l2': 10,
'num_threads':6
}
d_train = lgb.Dataset(X_ts,y_cs, weight=wts, free_raw_data=False)
 if not type(yt)is type(None):
d_cv = lgb.Dataset(xt,yt, free_raw_data=False, reference=d_train)
 model = lgb.train(params,d_train,num_boost_round=500,
valid_sets=d_cv,
verbose_eval=50)
 else :
model = lgb.train(params,d_train,num_boost_round=500)
 preds=model.predict(xtest)
 baggedpred+=preds
if not output_name is None:
joblib.dump(( model), output_name)

baggedpred/= estimators
return baggedpred 
<define_variables>
df_all['Title'].value_counts()
names=[]
for day in days_back_confimed:
names+=["days_ago_confirmed_count_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["ma" + str(window)+ "_count_confirmed" + str(k+1)for k in range(size)]
for day in days_back_fatalities:
names+=["days_ago_fatalitiescount_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["ma" + str(window)+ "_count_fatalities" + str(k+1)for k in range(size)]
names+=["confirmed_level"]
names+=["fatalities_level"]
if not extra_stable_columns is None and len(extra_stable_columns)>0:
names+=[k for k in extra_stable_columns]
if not group_names is None:
for gg in range(len(group_names)) :
for day in days_back_confimed_group:
names+=["days_ago_grooupcount_" + str(gg+1)+ "_" + str(day)]
for window in windows_group:
names+=["ma_group_" + str(gg+1)+ "_" + str(window)+ "_rate_" + str(k+1)for k in range(size_group)] 
<define_variables>
df_all.groupby('Title')['Survived'].mean()
<categorify>
df_all['Surname'] = df_all['Name'].str.split(',',expand=True)[0].str.strip()
df_train, df_test = divide_df(df_all)
 common_surname = [each for each in df_test['Surname'].unique() if each in df_train['Surname'].unique() ]
df_train.groupby('Surname')['Survived'].median()
train_survived_dict_surname = df_train[df_train['FamilySize']>1].groupby('Surname')['Survived'].median().to_dict()
new_dict_surname = {}
for key in common_surname:
if key in train_survived_dict_surname.keys() :
new_dict_surname[key] = train_survived_dict_surname[key]
df_train['Sur_rate'] = df_train['Surname'].map(new_dict_surname ).fillna(df_train['Survived'].mean())
 df_test['Sur_rate'] = df_test['Surname'].map(new_dict_surname ).fillna(df_train['Survived'].mean())
 df_train['Sur_count'] = df_train['Surname'].apply(lambda x: 1 if x in new_dict_surname.keys() else 0)
 df_test['Sur_count'] = df_test['Surname'].apply(lambda x: 1 if x in new_dict_surname.keys() else 0 )
def decay_4_first_10_then_1_f(array):
arr=[k for k in array]
for j in range(len(array)) :
if j<10:
arr[j]*=1./4.
 else :
arr[j]=0
return arr
	
def decay_16_first_10_then_1_f(array):
arr=[k for k in array]
for j in range(len(array)) :
if j<10:
arr[j]*=1./16.
 else :
arr[j]=0
return arr
def decay_2_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=1./2.
 return arr
def decay_4_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=1./4.
 return arr
	
def acceleratorx2_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=2.
 return arr
def decay_1_5_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=1./1.5
return arr
def decay_1_2_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=1./1.2
return arr
def decay_1_1_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=1./1.1
return arr
def stay_same_f(array):
arr=[0.0 for k in array]
return arr
def decay_2_last_12_linear_inter_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=1/2.
 arr12= arr[-12]
for j in range(0, 12):
arr[len(arr)-12 +j]= arr12/(j+1.)
return arr
def decay_1_5_last_12_linear_inter_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=1/(1.5)
 arr12= arr[-12]
for j in range(0, 12):
arr[len(arr)-12 +j]= arr12/(j+1.)
return arr
def decay_1_2_last_12_linear_inter_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=1./(1.2)
 arr12= arr[-12]
for j in range(0, 12):
arr[len(arr)-12 +j]= arr12/(j+1.)
return arr
def decay_4_last_12_linear_inter_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=1/4.
 arr12= arr[-12]
for j in range(0, 12):
arr[len(arr)-12 +j]= arr12/(j+1.)
return arr
def decay_8_last_12_linear_inter_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]*=1/8.
 arr12= arr[-12]
for j in range(0, 12):
arr[len(arr)-12 +j]= arr12/(j+1.)
return arr
def linear_last_12_f(array):
arr=[k for k in array]
for j in range(len(array)) :
arr[j]=array[j]
arr12= arr[-12]
for j in range(0, 12):
arr[len(arr)-12 +j]= arr12/(j+1.)
return arr
decay_4_first_10_then_1 =[ "Jilin_China","Tianjin_China"]
decay_4_first_10_then_1_fatality=[]
decay_16_first_10_then_1 =["Beijing_China","Fujian_China","Jiangsu_China"]
decay_16_first_10_then_1_fatality=[]
decay_4=["nan_Central African Republic","Shanghai_China","nan_Equatorial Guinea","nan_Korea, South","nan_Zambia"]
decay_4_fatality=["Heilongjiang_China","Inner Mongolia_China"]
decay_2 =["nan_Burundi","Faroe Islands_Denmark","nan_Eritrea","nan_Gabon","nan_Italy","nan_Brunei""Heilongjiang_China","Inner Mongolia_China",
"Shanxi_China","nan_Congo(Kinshasa)","nan_Cyprus","Reunion_France","Sint Maarten_Netherlands","nan_Angola"]
decay_2_fatality=["nan_Iran"]
stay_same=["nan_Diamond Princess"]
stay_same_fatality=["Beijing_China","Fujian_China","Qinghai_China","Jiangsu_China","Jilin_China","Macau_China","Ningxia_China","Tianjin_China"
,"Faroe Islands_Denmark","Shanxi_China"]
normal=[]
normal_fatality=["nan_Andorra","New South Wales_Australia","nan_Korea, South","New York_US","nan_Austria","Alberta_Canada","Nova Scotia_Canada",
"Saskatchewan_Canada","nan_Costa Rica","nan_Croatia","nan_Finland","Martinique_France","Mayotte_France","nan_Hungary","nan_Iceland","nan_Latvia","nan_Czechia"
,"nan_Luxembourg","nan_Malta","nan_Montenegro","nan_New Zealand","nan_Norway","nan_South Africa","Hawaii_US","Ohio_US","Pennsylvania_US","nan_Uruguay","nan_Venezuela",
"nan_West Bank and Gaza"]
decay_4_last_12_linear_inter =[ "nan_Dominica","New Caledonia_France",
"Saint Barthelemy_France","nan_Gambia","nan_Grenada","nan_Holy See",
"nan_Mauritania","nan_Nicaragua","nan_Saint Lucia","nan_Saint Vincent and the Grenadines",
"nan_Seychelles","nan_Suriname","nan_Mongolia",
"Montserrat_United Kingdom","Turks and Caicos Islands_United Kingdom", "Hong Kong_China","Northern Territory_Australia","Northwest Territories_Canada","Yukon_Canada"
,"Guangdong_China","nan_MS Zaandam","nan_Maldives","nan_Saint Kitts and Nevis","nan_Sao Tome and Principe","nan_South Sudan","nan_Western Sahara"]
decay_4_last_12_linear_inter_fatality=["nan_Somalia"]
decay_2_last_12_linear_inter =[ "nan_Chad","French Polynesia_France","nan_Laos","nan_Nepal","nan_Sudan","nan_Tanzania","Bermuda_United Kingdom","Cayman Islands_United Kingdom"
,"nan_Cabo Verde","nan_Eswatini","nan_Guyana","nan_Liberia","nan_Malawi","nan_Mozambique","nan_Somalia","nan_Timor-Leste","Falkland Islands(Malvinas)_United Kingdom",
"nan_Zimbabwe"]
decay_2_last_12_linear_inter_fatality=[]
decay_1_5 =["nan_Cambodia","nan_Croatia","nan_Denmark",
"nan_Bahamas","nan_Bahrain","nan_Barbados" ,"Manitoba_Canada","New Brunswick_Canada",
"Saskatchewan_Canada","nan_France","nan_Libya","nan_Malta",
	"Aruba_Netherlands","nan_Niger","nan_Spain","Virgin Islands_US",
	"Channel Islands_United Kingdom","Gibraltar_United Kingdom","nan_United Kingdom",'nan_Burma',"nan_Congo(Brazzaville)",
	"French Guiana_France","Martinique_France","Mayotte_France","nan_Georgia","nan_Iran","nan_New Zealand","nan_Syria",
	"nan_West Bank and Gaza"]
decay_1_5_fatality=["nan_Cuba","nan_Morocco","Ohio_US","Pennsylvania_US","Puerto Rico_US" ,"nan_Uzbekistan","nan_Cyprus","nan_France","nan_Niger","South Dakota_US"]
acceleratorx2=[]
acceleratorx2_fatality=["Newfoundland and Labrador_Canada"]
decay_1_5_last_12_linear_inter =["nan_Antigua and Barbuda","nan_Botswana","nan_Haiti","nan_Madagascar"]
decay_1_5_last_12_linear_inter_fatality =[]
decay_1_2 =["nan_Albania","nan_Andorra","New South Wales_Australia","nan_Austria","nan_Azerbaijan","nan_Bangladesh","nan_Belize","Alberta_Canada","nan_Mauritius","nan_Monaco","nan_Montenegro",
"Newfoundland and Labrador_Canada","nan_Costa Rica","nan_Czechia","nan_Dominican Republic","nan_Estonia","Guadeloupe_France","nan_Iceland","nan_Latvia","nan_Liechtenstein","nan_Luxembourg","nan_Norway",
"nan_Poland","nan_Rwanda","nan_South Africa","nan_Trinidad and Tobago","Arizona_US","California_US","Colorado_US","Connecticut_US","Georgia_US","Hawaii_US","Maine_US","New York_US","Oklahoma_US",
"Oregon_US","Pennsylvania_US","Tennessee_US","Washington_US","nan_Uruguay","nan_Venezuela","nan_Vietnam"]
decay_1_2_fatality=["nan_Afghanistan","nan_Belarus","nan_Djibouti","nan_Mali","nan_Mexico","nan_Peru","nan_Serbia","Kentucky_US","Puerto Rico_US","Texas_US","nan_Ukraine"]
decay_1_1 =["nan_Algeria","nan_Argentina","Tasmania_Australia","nan_Bolivia","Nova Scotia_Canada","nan_Finland","nan_Hungary","nan_Iraq",
"nan_Jordan","nan_Kyrgyzstan","nan_Malaysia","Alabama_US","Iowa_US","Michigan_US","Missouri_US","Nebraska_US","North Dakota_US ","Ohio_US","South Carolina_US"]
decay_1_1_fatality=["nan_Afghanistan","nan_Ecuador","nan_Kenya","nan_Korea, South","nan_Kosovo","Indiana_US","New Mexico_US","Oregon_US","Rhode Island_US","Tennessee_US","nan_United Arab Emirates"]
decay_1_2_last_12_linear_inter =[]
decay_1_2_last_12_linear_inter_fatality=["nan_Angola" ]
linear_last_12=["nan_Afghanistan","nan_Ireland","nan_Benin"]
linear_last_12_fatality=["nan_Guatemala","nan_Ireland"]
decay_8_last_12_linear_inter =[ "nan_Bhutan","Prince Edward Island_Canada","Greenland_Denmark","nan_Fiji","Saint Pierre and Miquelon_France","St Martin_France","nan_Namibia",
"Bonaire, Sint Eustatius and Saba_Netherlands","Curacao_Netherlands","nan_Papua New Guinea","Anguilla_United Kingdom","British Virgin Islands_United Kingdom",]
decay_8_last_12_linear_inter_fatality=[]
tr_frame=train_frame
features_train=tr_frame[names].values
standard_confirmed_train=tr_frame["ConfirmedCases"].values
standard_fatalities_train=tr_frame["Fatalities"].values
current_confirmed_train=tr_frame["ConfirmedCases"].values
features_cv=[]
name_cv=[]
standard_confirmed_cv=[]
standard_fatalities_cv=[]
names_=tr_frame["key"].values
training_horizon=int(features_train.shape[0]/len(unique_keys))
print("training horizon = ",training_horizon)
 for dd in range(training_horizon-1,features_train.shape[0],training_horizon):
features_cv.append(features_train[dd])
 name_cv.append(names_[dd])
 standard_confirmed_cv.append(standard_confirmed_train[dd])
 standard_fatalities_cv.append(standard_fatalities_train[dd])
 print(name_cv[-1], standard_confirmed_cv[-1], standard_fatalities_cv[-1])
 features_cv=np.array(features_cv)
 preds_confirmed_cv=np.zeros(( features_cv.shape[0],horizon))
preds_confirmed_standard_cv=np.zeros(( features_cv.shape[0],horizon))
preds_fatalities_cv=np.zeros(( features_cv.shape[0],horizon))
preds_fatalities_standard_cv=np.zeros(( features_cv.shape[0],horizon))
overal_rmsle_metric_confirmed=0.0
for j in range(preds_confirmed_cv.shape[1]):
this_features_cv=features_cv
preds=predict(features_cv, input_name=model_directory +"confirmedc"+ str(j))
preds_confirmed_cv[:,j]=preds
print(" modelling confirmed, case %d, , original cv %d and after %d "%(j,this_features_cv.shape[0],this_features_cv.shape[0]))
predictions=[]
for ii in range(preds_confirmed_cv.shape[0]):
current_prediction=standard_confirmed_cv[ii]
if current_prediction==0 :
current_prediction=0.1
this_preds=preds_confirmed_cv[ii].tolist()
name=name_cv[ii]
reserve=this_preds[0]
	
if name in normal:
this_preds=this_preds	
	
elif name in decay_4_first_10_then_1:
this_preds=decay_4_first_10_then_1_f(this_preds)
 		
elif name in decay_16_first_10_then_1:
this_preds=decay_16_first_10_then_1_f(this_preds)		
		
elif name in decay_4_last_12_linear_inter:
this_preds=decay_4_last_12_linear_inter_f(this_preds)		
elif name in decay_8_last_12_linear_inter:
this_preds=decay_8_last_12_linear_inter_f(this_preds)		
elif name in decay_4:
this_preds=decay_4_f(this_preds)
 elif name in decay_2:
this_preds=decay_2_f(this_preds)
 elif name in decay_2_last_12_linear_inter:
this_preds=decay_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_5:
this_preds=decay_1_5_f(this_preds)
 elif name in decay_1_5_last_12_linear_inter:
this_preds=decay_1_5_last_12_linear_inter_f(this_preds)
 elif name in decay_1_2:
this_preds=decay_1_2_f(this_preds)
 elif name in decay_1_2_last_12_linear_inter:
this_preds=decay_1_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_1:
this_preds=decay_1_1_f(this_preds)
 elif name in linear_last_12:
this_preds=linear_last_12_f(this_preds)
 elif name in acceleratorx2:
this_preds=acceleratorx2_f(this_preds)
 elif name in stay_same or "China" in name:
this_preds=stay_same_f(this_preds)
 for j in range(preds_confirmed_cv.shape[1]):
current_prediction+=np.expm1(max(0,this_preds[j]))
preds_confirmed_standard_cv[ii][j]=current_prediction
for j in range(preds_confirmed_cv.shape[1]):
this_features_cv=features_cv
preds=predict(features_cv, input_name=model_directory +"fatalc"+ str(j))
preds_fatalities_cv[:,j]=preds
print(" modelling fatalities, case %d, original cv %d and after %d "%(j,this_features_cv.shape[0],this_features_cv.shape[0]))
predictions=[]
for ii in range(preds_fatalities_cv.shape[0]):
current_prediction=standard_fatalities_cv[ii]
this_preds=preds_fatalities_cv[ii].tolist()
name=name_cv[ii]
reserve=this_preds[0]
	
if name in normal_fatality:
this_preds=this_preds	 	
	
elif name in decay_4_first_10_then_1_fatality:
this_preds=decay_4_first_10_then_1_f(this_preds)
 		
elif name in decay_16_first_10_then_1_fatality:
this_preds=decay_16_first_10_then_1_f(this_preds)
 elif name in decay_4_last_12_linear_inter_fatality:
this_preds=decay_4_last_12_linear_inter_f(this_preds)		
elif name in decay_8_last_12_linear_inter_fatality:
this_preds=decay_8_last_12_linear_inter_f(this_preds)		
elif name in decay_4_fatality:
this_preds=decay_4_f(this_preds)		
		
elif name in decay_2_fatality:
this_preds=decay_2_f(this_preds)
 elif name in decay_2_last_12_linear_inter_fatality:
this_preds=decay_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_5_fatality:
this_preds=decay_1_5_f(this_preds)	
elif name in decay_1_5_last_12_linear_inter_fatality:
this_preds=decay_1_5_last_12_linear_inter_f(this_preds)
 elif name in decay_1_2_fatality:
this_preds=decay_1_2_f(this_preds)
 elif name in decay_1_2_last_12_linear_inter_fatality:
this_preds=decay_1_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_1_fatality:
this_preds=decay_1_1_f(this_preds)
 elif name in linear_last_12_fatality:
this_preds=linear_last_12_f(this_preds)
 elif name in acceleratorx2_fatality:
this_preds=acceleratorx2_f(this_preds)
 elif name in stay_same_fatality:
this_preds=stay_same_f(this_preds)
 elif name in normal:
this_preds=this_preds	
	
elif name in decay_4_first_10_then_1:
this_preds=decay_4_first_10_then_1_f(this_preds)
 		
elif name in decay_16_first_10_then_1:
this_preds=decay_16_first_10_then_1_f(this_preds)		
elif name in decay_4_last_12_linear_inter:
this_preds=decay_4_last_12_linear_inter_f(this_preds)		
elif name in decay_8_last_12_linear_inter:
this_preds=decay_8_last_12_linear_inter_f(this_preds)		
elif name in decay_4:
this_preds=decay_4_f(this_preds)
 		
elif name in decay_2:
this_preds=decay_2_f(this_preds)
 elif name in decay_2_last_12_linear_inter:
this_preds=decay_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_5:
this_preds=decay_1_5_f(this_preds)
 elif name in decay_1_5_last_12_linear_inter:
this_preds=decay_1_5_last_12_linear_inter_f(this_preds)
 elif name in decay_1_2:
this_preds=decay_1_2_f(this_preds)
 elif name in decay_1_2_last_12_linear_inter:
this_preds=decay_1_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_1:
this_preds=decay_1_1_f(this_preds)
 elif name in linear_last_12:
this_preds=linear_last_12_f(this_preds)
 elif name in acceleratorx2:
this_preds=acceleratorx2_f(this_preds)
 elif name in stay_same or "China" in name:
this_preds=stay_same_f(this_preds)
 for j in range(preds_fatalities_cv.shape[1]):
current_prediction+=np.expm1(max(0,this_preds[j]))
preds_fatalities_standard_cv[ii][j]=current_prediction<categorify>
common_ticket = [each for each in df_test['Ticket'].unique() if each in df_train['Ticket'].unique() ]
train_survived_dict_ticket = df_train[df_train['Ticket_Frequency']>1].groupby('Ticket')['Survived'].median().to_dict()
new_dict_ticket = {}
for key in common_ticket:
if key in train_survived_dict_ticket.keys() :
new_dict_ticket[key] = train_survived_dict_ticket[key]
df_train['tic_rate'] = df_train['Ticket'].map(new_dict_surname ).fillna(df_train['Survived'].mean())
 df_test['tic_rate'] = df_test['Ticket'].map(new_dict_surname ).fillna(df_train['Survived'].mean())
 df_train['tic_count'] = df_train['Ticket'].apply(lambda x: 1 if x in new_dict_ticket.keys() else 0)
 df_test['tic_count'] = df_test['Ticket'].apply(lambda x: 1 if x in new_dict_ticket.keys() else 0 )
key_to_confirmed_rate={}
key_to_fatality_rate={}
key_to_confirmed={}
key_to_fatality={}
print(len(features_cv), len(name_cv),len(standard_confirmed_cv),len(standard_fatalities_cv))
print(preds_confirmed_cv.shape,preds_confirmed_standard_cv.shape,preds_fatalities_cv.shape,preds_fatalities_standard_cv.shape)
 for j in range(len(name_cv)) :
key_to_confirmed_rate[name_cv[j]]=preds_confirmed_cv[j,:].tolist()
key_to_fatality_rate[name_cv[j]]=preds_fatalities_cv[j,:].tolist()
key_to_confirmed[name_cv[j]] =preds_confirmed_standard_cv[j,:].tolist()
key_to_fatality[name_cv[j]]=preds_fatalities_standard_cv[j,:].tolist() 
<merge>
for df in [df_train, df_test]:
df['Survival_Rate'] =(df['tic_rate'] + df['Sur_rate'])/ 2
df['Survival_Rate_NA'] =(df['tic_count'] + df['Sur_count'])/ 2
train_new=train[["Date","ConfirmedCases","Fatalities","key","rate_ConfirmedCases","rate_Fatalities"]]
test_new_count=pd.merge(test,train_new, how="left", left_on=["key","Date"], right_on=["key","Date"] ).reset_index(drop=True)
 test_new_count<categorify>
df_all = concat_df(df_train,df_test )
def fillin_columns(frame,key_column, original_name, training_horizon, test_horizon, unique_values, key_to_values):
keys=frame[key_column].values
original_values=frame[original_name].values.tolist()
print(len(keys), len(original_values), training_horizon ,test_horizon,len(key_to_values))
for j in range(unique_values):
current_index=(j *(training_horizon +test_horizon)) +training_horizon
current_key=keys[current_index]
values=key_to_values[current_key]
co=0
for g in range(current_index, current_index + test_horizon):
original_values[g]=values[co]
co+=1
frame[original_name]=original_values
all_days=int(test_new.shape[0]/len(unique_keys))
tr_horizon=all_days-horizon
print(all_days,tr_horizon, horizon)
 fillin_columns(test_new_count,"key", 'ConfirmedCases', tr_horizon, horizon, len(unique_keys), key_to_confirmed)
 fillin_columns(test_new_count,"key", 'Fatalities', tr_horizon, horizon, len(unique_keys), key_to_fatality)
 fillin_columns(test_new_count,"key", 'rate_ConfirmedCases', tr_horizon, horizon, len(unique_keys), key_to_confirmed_rate)
 fillin_columns(test_new_count,"key", 'rate_Fatalities', tr_horizon, horizon, len(unique_keys), key_to_fatality_rate)
 test_dat=test_new['Fatalities'].values.tolist()
test_new_countfat=test_new_count['Fatalities'].values.tolist()
for j in range(len(test_dat)) :
if test_dat[j]<1:
test_dat[j]=test_new_countfat[j]
test_new2= test_new.copy()
test_new2['Fatalities']=test_dat
test_new2 
<save_to_csv>
drop_list.append('tic_rate')
 drop_list.append('Sur_rate')
 drop_list.append('tic_count')
 drop_list.append('Sur_count' )
submission=test_new2[["ForecastId","ConfirmedCases","Fatalities"]]
submission.to_csv("submission.csv", index=False)
<compute_test_metric>
df_all.drop(drop_list,inplace=True, axis=1)
 drop_list = []
%matplotlib inline
def sigmoid_sqrt_func(x, a, b, c, d, e):
return c + d /(1.0 + np.exp(-a*x+b)) + e*x**0.5
def sigmoid_linear_func(x, a, b, c, d, e):
return c + d /(1.0 + np.exp(-a*x+b)) + e*0.1*x
def sigmoid_quad_func(x, a, b, c, d, e, f):
return c + d /(1.0 + np.exp(-a*x+b)) + e*0.1*x + f*0.001*x*x
def sigmoid_func(x, a, b, c, d):
return c + d /(1.0 + np.exp(-a*x+b))
def exp_func(x, a, b, c, d):
return c + d * np.exp(a*x+b)
 def func_fitting(y, func=sigmoid_func, x_scale=50.0, y_scale=10000.0, start_pred=8, AN=0, MAXN=60, PN=15, b=5):
x = range(len(y))
x_real = np.array(x)/x_scale
y_real = np.array(y)/y_scale
x_train = x_real
y_train = y_real
def next_day_pred(AN, BN):
x_train = x_real[AN:BN]
y_train = y_real[AN:BN]
popt, pcov = curve_fit(func, x_train, y_train,
method='trf',
maxfev=20000,
p0=(1, 0, 0, 1),
bounds=[(-b, -np.inf, -np.inf, -b),(b, np.inf, np.inf, b)], 
)
 x_pred = np.array(range(MAXN)) /x_scale
y_pred = func(x_pred, *popt)
 return x_pred, y_pred
NP = start_pred
y_pred = [np.nan]*NP
y_pred_list = []
for BN in range(NP, len(y_real)) :
x_pred, y_pred_ = next_day_pred(BN-PN, BN)
 y_pred.append(y_pred_[BN])
 y_pred_list.append(y_pred_)
 for BN in range(len(y_real), len(y_pred_)) :
y_pred.append(y_pred_[BN])
 y_pred = np.array(y_pred)
 y_pred_list = np.array(y_pred_list)
 y_pred_std = np.std(y_pred_list[-2:], axis=0)
 return x_real*x_scale, y_real*y_scale, x_train*x_scale, y_train*y_scale, \
x_pred*x_scale, y_pred*y_scale, y_pred_std*y_scale
def draw_figure(start_date, title, x_real, y_real, x_train, y_train, x_pred, y_pred, y_pred_std):
def to_date(idx):
idx = np.round(idx)
 return datetime.datetime.strptime(start_date, '%m/%d/%Y' ).date() + datetime.timedelta(days=idx)
 fig, ax1 = plt.subplots(figsize=[14, 7])
 plot1 = ax1.plot(list(map(to_date, x_real)) , y_real, 'gs',label='original')
 plot2 = ax1.plot(list(map(to_date, x_pred)) , y_pred, 'r',label='predict')
 plot3 = ax1.fill_between(list(map(to_date, x_pred)) ,
np.maximum(0,(y_pred-y_pred_std)) , 
(y_pred+y_pred_std),
alpha=0.2, edgecolor='
plot0 = ax1.plot(list(map(to_date, x_train)) , y_train, 'y.',label='history')
 ax2=ax1.twinx()
ax2.plot(list(map(to_date, x_real)) [1:],(y_real[1:]-y_real[:-1]), '-s',label='original add')
 ax2.plot(list(map(to_date, x_pred)) [1:],(y_pred[1:]-y_pred[:-1]), '-',label='pred add')
 plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%m-%d'))
plt.gca().xaxis.set_major_locator(mdates.DayLocator())
 plt.gcf().autofmt_xdate()
plt.xlabel('x')
 plt.ylabel('y')
 fig.legend(loc=2)
 plt.title(title)
 plt.savefig('{}.pdf'.format(title))
plt.show()
date = list(map(to_date, x_pred))
pred = y_pred
real = y_real
for i in range(len(pred)) :
if i < len(real):
print('{}\t{:.0f}\t{:.0f}\t{:.3f}'.format(date[i], real[i], pred[i], np.abs(pred[i]-real[i])/real[i]*100))
else:
print('{}\t-\t{:.0f}'.format(date[i], pred[i]))
return pred<load_from_csv>
drop_list.append('Name')
 df_all.drop(drop_list,inplace=True, axis=1)
 drop_list = []
train_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 pred_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv')
 train_data = train_data.fillna(value='NULL')
 test_data = test_data.fillna(value='NULL' )<count_unique_values>
drop_list.append('Ticket')
 drop_list.append('Surname')
 df_all.drop(drop_list,inplace=True, axis=1)
 drop_list = []
train_date_list = train_data.iloc[:, 3].unique()
print(len(train_date_list))
print(train_date_list)
 test_date_list = test_data.iloc[:, 3].unique()
print(len(test_date_list))
print(test_date_list)
 len(train_data.groupby(['Province_State', 'Country_Region']))
len(test_data.groupby(['Province_State', 'Country_Region']))<load_from_csv>
def non_numeric_features(df):
alist = []
for col in df.columns:
if 'float' not in str(df[col].dtype)and 'int' not in str(df[col].dtype):
alist.append(col)
 return alist
non_num_list = non_numeric_features(df_all )
start_date = '01/22/2020'
start_pred = 81
start_submit = 71
len_pred = 30
test_date_list = test_data.iloc[:, 3].unique()
test_data_filled = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 test_data_filled = test_data_filled.fillna(value='NULL')
 test_data_filled['ConfirmedCases'] = pred_data['ConfirmedCases']
test_data_filled['Fatalities'] = pred_data['Fatalities']
for idx,(k, v)in enumerate(train_data.groupby(['Province_State', 'Country_Region'])) :
print(idx, k, end=' ')
 b_cc, b_f = 5, 3
if k[1] == 'China':
b_cc = 1
b_f = 1
if k[0] == 'Hong Kong':
b_cc = 3
elif k[1] == 'Italy':
b_cc = 10
b_f = 3
elif k[1] == 'US':
b_cc = 5
b_f = 3
elif k[1] == 'Spain':
b_cc = 4
b_f = 3
hist_num = v.loc[:,'ConfirmedCases'].tolist()
ret = func_fitting(hist_num, y_scale=max(1000, np.max(hist_num)) , b=b_cc,
start_pred=start_pred, PN=10, MAXN=len(hist_num)+len_pred)
 ret = list(ret)
 real_cc = np.round(np.array(ret[1]))
pred_cc = np.round(np.array(ret[5]))
for i in range(len(real_cc)) :
pred_cc[i] = real_cc[i]
pred_cc = pred_cc[start_submit:]
print(pred_cc)
 hist_num = v.loc[:,'Fatalities'].tolist()
ret = func_fitting(hist_num, y_scale=max(1000, np.max(hist_num)) , b=b_f,
start_pred=start_pred, PN=10, MAXN=len(hist_num)+len_pred)
 ret = list(ret)
 real_f = np.round(np.array(ret[1]))
pred_f = np.round(np.array(ret[5]))
for i in range(len(real_f)) :
pred_f[i] = real_f[i]
pred_f = pred_f[start_submit:]
print(pred_f)
 for i in range(14, len(pred_cc)) :
if pred_f[i] < 20 and pred_cc[i] > 200 and k[1] != 'China':
pred_f[i] = pred_cc[i] * 0.01 * np.log10(pred_cc[i])
 print(pred_f)
 print(pred_cc[-1], pred_f[-1])
 for i in range(len(pred_cc)) :
index =(test_data_filled['Province_State'] == k[0])& \ 
(test_data_filled['Country_Region'] == k[1])& \ 
(test_data_filled['Date'] == test_date_list[i])
 test_data_filled.loc[index, 'ConfirmedCases'] = pred_cc[i]
test_data_filled.loc[index, 'Fatalities'] = pred_f[i]<filter>
df_train, df_test = divide_df(df_all)
 dfs =[df_train,df_test]
for df in dfs:
for feature in non_num_list:
df[feature] = LabelEncoder().fit_transform(df[feature] )
submission = test_data_filled.loc[:,['ForecastId', 'ConfirmedCases', 'Fatalities']]<save_to_csv>
cat_features = ['Embarked', 'Sex', 'Family', 'Title','Pclass','Deck']
for i,df in enumerate(dfs):
for feature in cat_features:
df = pd.concat([df,pd.get_dummies(df[feature], prefix=feature)],axis=1 ).drop([feature],axis=1)
 dfs[i] = df
submission.to_csv("submission.csv", index=False)
 submission.head(500 )<set_options>
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier, ExtraTreesClassifier
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold
py.init_notebook_mode(connected=True)
 pio.templates.default = "plotly_dark"
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
df_train = dfs[0]
df_train.name = 'training'
df_test = dfs[1]
df_test.name = 'testing'
display_missing(df_train)
 display_missing(df_test )
train_df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test_df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 submission_df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv')
 train_df = train_df.drop(['Id'],axis=1)
 train_df.rename(columns={'Country_Region':'Country'}, inplace=True)
 test_df.rename(columns={'Country_Region':'Country'}, inplace=True)
 train_df['Province_State'].fillna("None*", inplace = True)
 test_df['Province_State'].fillna("None*", inplace = True)
 renameCountryNames = {
"Congo(Brazzaville)": "Congo1",
"Congo(Kinshasa)": "Congo2",
"Cote d'Ivoire": "Côte d'Ivoire",
"Czechia": "Czech Republic(Czechia)",
"Korea, South": "South Korea",
"Saint Kitts and Nevis": "Saint Kitts & Nevis",
"Saint Vincent and the Grenadines": "St.Vincent & Grenadines",
"Taiwan*": "Taiwan",
"US": "United States"
}
train_df.replace({'Country': renameCountryNames}, inplace=True)
 test_df.replace({'Country': renameCountryNames}, inplace=True)
 specific_countries = ['United States', 'United Kingdom','Netherlands']
days_df = train_df['Date'].apply(lambda dt: datetime.datetime.strptime(dt, '%Y-%m-%d')- datetime.datetime.strptime('2020-01-21', '%Y-%m-%d')).apply(lambda x : str(x ).split() [0] ).astype(int)
 train_df['Days'] = days_df
days_df = test_df['Date'].apply(lambda dt: datetime.datetime.strptime(dt, '%Y-%m-%d')- datetime.datetime.strptime('2020-01-21', '%Y-%m-%d')).apply(lambda x : str(x ).split() [0] ).astype(int)
 test_df['Days'] = days_df
all_countries = train_df['Country'].unique()
display(train_df.tail())
 display(test_df.head())
 ww_df = train_df.groupby('Date')[['ConfirmedCases', 'Fatalities']].sum().reset_index()
ww_df['new_case'] = ww_df['ConfirmedCases'] - ww_df['ConfirmedCases'].shift(1)
 ww_df['new_deaths'] = ww_df['Fatalities'] - ww_df['Fatalities'].shift(1)
 country_df = train_df.groupby(['Date', 'Country'])[['ConfirmedCases', 'Fatalities']].sum().reset_index()
target_date = country_df['Date'].max()
train_end_day = train_df['Days'].max()
test_start_day = test_df['Days'].min()
test_end_day = test_df['Days'].max()
display(country_df[country_df['Country']=='France'][80:] )<groupby>
X_train = StandardScaler().fit_transform(df_train.drop('Survived',axis=1))
y_train = df_train['Survived']
X_test = StandardScaler().fit_transform(df_test)
 print('X_train shape: {}'.format(X_train.shape))
print('y_train shape: {}'.format(y_train.shape))
print('X_test shape: {}'.format(X_test.shape))
country_province_df = train_df[train_df['Country']=='United States'].groupby(['Date', 'Province_State'])[['ConfirmedCases', 'Fatalities']].sum().reset_index()
top_province_df = country_province_df.query('(Date == @target_date)' ).sort_values('ConfirmedCases', ascending=False)
 top30_provinces = top_province_df.sort_values('ConfirmedCases', ascending=False ).iloc[:30]['Province_State'].unique()
country_province_df['prev_cases'] = country_province_df.groupby('Province_State')['ConfirmedCases'].shift(1)
 country_province_df['New Case'] = country_province_df['ConfirmedCases'] - country_province_df['prev_cases']
country_province_df['New Case'].fillna(0, inplace=True)
 country_province_df['prev_deaths'] = country_province_df.groupby('Province_State')['Fatalities'].shift(1)
 country_province_df['New Death'] = country_province_df['Fatalities'] - country_province_df['prev_deaths']
country_province_df['New Death'].fillna(0, inplace=True)
 display(country_province_df[-30:])
 for province in top30_provinces:
present_country_df = country_province_df[country_province_df['Province_State']==province]
px.bar(present_country_df,
x='Date', y='New Case', color='Province_State',
title=f'United States : DAILY NEW Confirmed cases in '+province ).show()<create_dataframe>
single_best_model = RandomForestClassifier(criterion='gini',
n_estimators=1100,
max_depth=5,
min_samples_split=4,
min_samples_leaf=5,
max_features='auto',
oob_score=True,
random_state=SEED,
n_jobs=-1,
verbose=1)
def get_time_series(df,country_name,insert=False):
if df[df['Country'] == country_name]['Province_State'].nunique() > 1:
country_table = df[df['Country'] == country_name]
if insert:
country_df = pd.DataFrame(pd.pivot_table(country_table, values = ['ConfirmedCases','Fatalities','Days'],
index='Date', aggfunc=sum ).to_records())
 return country_df.set_index('Date')[['ConfirmedCases','Fatalities']]
return country_table.set_index('Date')[['Province_State','ConfirmedCases','Fatalities','Days']]
df = df[(df['Country'] == country_name)]
return df.set_index('Date')[['ConfirmedCases','Fatalities','Days']]
def get_time_series_province(province):
df = full_table[(full_table['Province_State'] == province)]
return df.set_index('Date')[['ConfirmedCases','Fatalities']]
province_country_dfs = {}
no_province_country_dfs = {}
absent_country_in_age_data_dfs = {}
province_country_dfs_list=[]
no_province_country_dfs_list=[]
absent_country_in_age_data_dfs_list=[]
province_countries=train_df[train_df['Province_State']!="None*"]['Country'].unique()
no_province_countries=train_df[train_df['Province_State']=="None*"]['Country'].unique()
no_province_countries= [x for x in no_province_countries if x not in province_countries]
for country in province_countries:
province_country_dfs[country] = get_time_series(train_df,country)
 for country in no_province_countries:
no_province_country_dfs[country] = get_time_series(train_df,country)
 print([x for x in no_province_countries if x in top30_countries])
 print([x for x in province_countries if x in top30_countries])
 assert(len([x for x in all_countries if x not in list(no_province_countries)+list(province_countries)])==0)
 display(province_country_dfs['United States'] )<import_modules>
single_best_model.fit(X_train, y_train)
 predictions1 = single_best_model.predict(X_test )
print(tf.__version__ )<feature_engineering>
submission_df = pd.DataFrame(columns=['PassengerId', 'Survived'])
 submission_df['PassengerId'] = passengerId
submission_df['Survived'] = predictions1.astype(int)
 submission_df.to_csv('my_submissions.csv', header=True, index=False)
 submission_df.head(10 )
<prepare_x_and_y><EOS> 
com = pd.DataFrame({'pre1':predictions1, 'pre2': predictions2} )