Kaggle competition: COVID19 Global Forecasting (Week 4)
model.load_weights('.. /input/tpu-training-tensorflow-iafoos-method-42x256x256x3/efficientnet-b0.h5' )<load_from_csv>
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5
def fix_target(frame, key, target, new_target_name="target"):
corrections = 0
group_keys = frame[ key].values.tolist()
target = frame[target].values.tolist()
for i in range(1, len(group_keys)- 1):
previous_group = group_keys[i - 1]
current_group = group_keys[i]
previous_value = target[i - 1]
current_value = target[i]
if current_group == previous_group:
if current_value<previous_value:
current_value=previous_value
target[i] =current_value
target[i] =max(0,target[i])
 frame[new_target_name] = np.array(target)
 def rate(frame, key, target, new_target_name="rate"):
corrections = 0
group_keys = frame[ key].values.tolist()
target = frame[target].values.tolist()
rate=[1.0 for k in range(len(target)) ]
for i in range(1, len(group_keys)- 1):
previous_group = group_keys[i - 1]
current_group = group_keys[i]
previous_value = target[i - 1]
current_value = target[i]
if current_group == previous_group:
if previous_value!=0.0:
rate[i]=current_value/previous_value
rate[i] =max(1,rate[i])
 frame[new_target_name] = np.array(rate)
 def get_data_by_key(dataframe, key, key_value, fields=None):
mini_frame=dataframe[dataframe[key]==key_value]
if not fields is None:
mini_frame=mini_frame[fields].values
return mini_frame
directory="/kaggle/input/covid19-global-forecasting-week-4/"
train=pd.read_csv(directory + "train.csv", parse_dates=["Date"] , engine="python")
 test=pd.read_csv(directory + "test.csv", parse_dates=["Date"], engine="python")
 train["key"]=train[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 test["key"]=test[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 max_train_date=train["Date"].max()
max_test_date=test["Date"].max()
horizon=(max_test_date-max_train_date ).days
print("horizon", int(horizon))
target1="ConfirmedCases"
target2="Fatalities"
key="key"
TRAIN = '.. /input/prostate-cancer-grade-assessment/train_images/'
MASKS = '.. /input/prostate-cancer-grade-assessment/train_label_masks/'
BASE_PATH = '.. /input/prostate-cancer-grade-assessment/'
train = pd.read_csv(BASE_PATH + "train.csv")
 train.head()<load_from_csv>
fix_target(train, key, target1, new_target_name=target1)
 fix_target(train, key, target2, new_target_name=target2)
 rate(train, key, target1, new_target_name="rate_" +target1)
 rate(train, key, target2, new_target_name="rate_" +target2)
 unique_keys=train[key].unique()
print(len(unique_keys))
train
sub = pd.read_csv(".. /input/prostate-cancer-grade-assessment/sample_submission.csv")
 sub.head()<load_from_csv>
def get_lags(rate_array, current_index, size=20):
lag_confirmed_rate=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
lag_confirmed_rate[j]=rate_array[current_index-j]
else :
break
return lag_confirmed_rate
def days_ago_thresold_hit(full_array, indx, thresold):
days_ago_confirmed_count_10=-1
if full_array[indx]>thresold:
for j in range(indx,-1,-1):
entered=False
if full_array[j]<=thresold:
days_ago_confirmed_count_10=abs(j-indx)
 entered=True
break
if entered==False:
days_ago_confirmed_count_10=100
return days_ago_confirmed_count_10
def ewma_vectorized(data, alpha):
sums=sum([(alpha**(k+1)) *data[k] for k in range(len(data)) ])
 counts=sum([(alpha**(k+1)) for k in range(len(data)) ])
 return sums/counts
def generate_ma_std_window(rate_array, current_index, size=20, window=3):
ma_rate_confirmed=[-1 for k in range(size)]
std_rate_confirmed=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
ma_rate_confirmed[j]=np.mean(rate_array[max(0,current_index-j-window+1):current_index-j+1])
 std_rate_confirmed[j]=np.std(rate_array[max(0,current_index-j-window+1):current_index-j+1])
 else :
break
return ma_rate_confirmed, std_rate_confirmed
def generate_ewma_window(rate_array, current_index, size=20, window=3, alpha=0.05):
ewma_rate_confirmed=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
ewma_rate_confirmed[j]=ewma_vectorized(rate_array[max(0,current_index-j-window+1):current_index-j+1, ], alpha)
 else :
break
return ewma_rate_confirmed
def get_target(rate_col, indx, horizon=33, average=3, use_hard_rule=False):
target_values=[-1 for k in range(horizon)]
cou=0
for j in range(indx+1, indx+1+horizon):
if j<len(rate_col):
if average==1:
target_values[cou]=rate_col[j]
else :
if use_hard_rule and j +average <=len(rate_col):
target_values[cou]=np.mean(rate_col[j:j +average])
 else :
target_values[cou]=np.mean(rate_col[j:min(len(rate_col),j +average)])
 cou+=1
else :
break
return target_values
def dereive_features(frame, confirmed, fatalities, rate_confirmed, rate_fatalities,
horizon ,size=20, windows=[3,7], days_back_confimed=[1,10,100], days_back_fatalities=[1,2,10]):
targets=[]
names=["lag_confirmed_rate" + str(k+1)for k in range(size)]
for day in days_back_confimed:
names+=["days_ago_confirmed_count_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["std" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["ewma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["lag_fatalities_rate" + str(k+1)for k in range(size)]
for day in days_back_fatalities:
names+=["days_ago_fatalitiescount_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["std" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["ewma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["confirmed_level"]
names+=["fatalities_level"]
names+=["confirmed_plus" + str(k+1)for k in range(horizon)]
names+=["fatalities_plus" + str(k+1)for k in range(horizon)]
features=[]
for i in range(len(confirmed)) :
row_features=[]
lag_confirmed_rate=get_lags(rate_confirmed, i, size=size)
 row_features+=lag_confirmed_rate
for day in days_back_confimed:
days_ago_confirmed_count_10=days_ago_thresold_hit(confirmed, i, day)
 row_features+=[days_ago_confirmed_count_10]
for window in windows:
ma3_rate_confirmed,std3_rate_confirmed= generate_ma_std_window(rate_confirmed, i, size=size, window=window)
 row_features+= ma3_rate_confirmed
row_features+= std3_rate_confirmed
ewma3_rate_confirmed=generate_ewma_window(rate_confirmed, i, size=size, window=window, alpha=0.05)
 row_features+= ewma3_rate_confirmed
lag_fatalities_rate=get_lags(rate_fatalities, i, size=size)
 row_features+=lag_fatalities_rate
for day in days_back_fatalities:
days_ago_fatalitiescount_2=days_ago_thresold_hit(fatalities, i, day)
 row_features+=[days_ago_fatalitiescount_2]
for window in windows:
ma3_rate_fatalities,std3_rate_fatalities= generate_ma_std_window(rate_fatalities, i, size=size, window=window)
 row_features+= ma3_rate_fatalities
row_features+= std3_rate_fatalities
ewma3_rate_fatalities=generate_ewma_window(rate_fatalities, i, size=size, window=window, alpha=0.05)
 row_features+= ewma3_rate_fatalities
confirmed_level=0

 confirmed_level= confirmed[i]
row_features+=[confirmed_level]
fatalities_is_level=0

 fatalities_is_level= fatalities[i]
row_features+=[fatalities_is_level]
confirmed_plus=get_target(rate_confirmed, i, horizon=horizon)
 row_features+= confirmed_plus
fatalities_plus=get_target(rate_fatalities, i, horizon=horizon)
 row_features+= fatalities_plus
features.append(row_features)
 new_frame=pd.DataFrame(data=features, columns=names ).reset_index(drop=True)
 frame=frame.reset_index(drop=True)
 frame=pd.concat([frame, new_frame], axis=1)
 return frame
def feature_engineering_for_single_key(frame, group, key, horizon=33, size=20, windows=[3,7],
days_back_confimed=[1,10,100], days_back_fatalities=[1,2,10]):
mini_frame=get_data_by_key(frame, group, key, fields=None)
 mini_frame_with_features=dereive_features(mini_frame, mini_frame["ConfirmedCases"].values,
mini_frame["Fatalities"].values, mini_frame["rate_ConfirmedCases"].values,
mini_frame["rate_Fatalities"].values, horizon ,size=size, windows=windows,
days_back_confimed=days_back_confimed, days_back_fatalities=days_back_fatalities)
 return mini_frame_with_features
test = pd.read_csv(".. /input/prostate-cancer-grade-assessment/test.csv")
 test.head()<define_variables>
def bagged_set_train(X_ts,y_cs,wts, seed, estimators,xtest, xt=None,yt=None, output_name=None):
baggedpred=np.array([ 0.0 for d in range(0, xtest.shape[0])])
 for n in range(0, estimators):
params = {'objective': 'rmse',
'metric': 'rmse',
'boosting': 'gbdt',
'learning_rate': 0.01,
'drop_rate':0.01,
'skip_drop':0.6,
'uniform_drop':True,
'verbose': -1,
'num_leaves': 30,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'bagging_seed': seed + n,
'feature_fraction': 0.8,
'feature_fraction_seed': seed + n,
'min_data_in_leaf': 10,
'max_bin': 100,
'max_depth':20,
'reg_alpha':1,
'lambda_l2': 10,
'num_threads':6
}
d_train = lgb.Dataset(X_ts,y_cs, weight=wts, free_raw_data=False)
 if not type(yt)is type(None):
d_cv = lgb.Dataset(xt,yt, free_raw_data=False, reference=d_train)
 model = lgb.train(params,d_train,num_boost_round=500,
valid_sets=d_cv,
verbose_eval=50)
 else :
model = lgb.train(params,d_train,num_boost_round=500)
 preds=model.predict(xtest)
 baggedpred+=preds
if not output_name is None:
joblib.dump(( model), output_name)

baggedpred/= estimators
return baggedpred
TEST = '.. /input/prostate-cancer-grade-assessment/test_images/'<define_variables>
names=["lag_confirmed_rate" + str(k+1)for k in range(size)]
for day in days_back_confimed:
names+=["days_ago_confirmed_count_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["std" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["ewma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["lag_fatalities_rate" + str(k+1)for k in range(size)]
for day in days_back_fatalities:
names+=["days_ago_fatalitiescount_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["std" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["ewma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["confirmed_level"]
names+=["fatalities_level"]
PRED_PATH = TEST
df = sub
t_df = test<concatenate>
number_of_folds=5
seed=1412
target_confirmed=["confirmed_plus" + str(k+1)for k in range(horizon)]
target_fatalities=["fatalities_plus" + str(k+1)for k in range(horizon)]
rmsle_metric_confirmed=0.0
rmsle_metric_fatalities=0.0
print(train_frame[target1].mean())
 print(len(train_frame['key'].unique()))
number_of_folds=0
def decay_4_first_10_then_1_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
if j<10:
arr[j]=1.+(max(1,array[j])-1.) /4.
 else :
arr[j]=1.
 return arr
def decay_2_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /2.
 return arr
def decay_1_5_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /1.5
return arr
def stay_same_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.
 return arr
def decay_2_last_12_linear_inter_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /2.
 arr12=(max(1,arr[-12])-1.) /12.
 for j in range(0, 12):
arr[len(arr)-12 +j]= max(1, 1 +(( arr12*12)-(j+1)*arr12))
return arr
def linear_last_12_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=max(1,array[j])
 arr12=(max(1,arr[-12])-1.) /12.
 for j in range(0, 12):
arr[len(arr)-12 +j]= max(1, 1 +(( arr12*12)-(j+1)*arr12))
return arr
decay_4_first_10_then_1 =["Beijing_China","Fujian_China","Guangdong_China", "Hong Kong_China",
"Inner Mongolia_China","Jiangsu_China","Liaoning_China","Macau_China","Shandong_China","Tianjin_China",
"Yunnan_China","Zhejiang_China","Northern Territory_Australia","nan_Bahamas",
"nan_Belize","nan_Benin","nan_Bhutan","nan_Seychelles"]
decay_2 =["Shanghai_China" , "nan_Afghanistan","nan_Andorra","Australian Capital Territory_Australia",
"South Australia_Australia","Tasmania_Australia","nan_Azerbaijan","nan_Bahrain","nan_Bangladesh","nan_Belarus"
"nan_Belgium","nan_Bolivia","nan_Cameroon","Manitoba_Canada","New Brunswick_Canada","Newfoundland and Labrador_Canada",
"Saskatchewan_Canada","nan_Central African Republic","nan_Congo(Kinshasa)","nan_Cote d'Ivoire","nan_Cuba","Mayotte_France",
"nan_Honduras"]
decay_2_last_12_linear_inter=["nan_Angola" , "nan_Barbados","nan_Cabo Verde" ,"Prince Edward Island_Canada","nan_Chad",
"nan_Congo(Brazzaville)","Greenland_Denmark","nan_Djibouti","nan_Dominica","nan_El Salvador","nan_Equatorial Guinea",
"nan_Eritrea","nan_Eswatini","nan_Fiji","French Guiana_France","French Polynesia_France","New Caledonia_France",
"Saint Barthelemy_France","St Martin_France","nan_Gabon","nan_Gambia","nan_Grenada","nan_Guinea","nan_Guinea-Bissau",
"nan_Guyana","nan_Haiti","nan_Holy See","nan_Kyrgyzstan","nan_Laos","nan_Liberia","nan_Libya","nan_Madagascar",
"nan_Maldives","nan_Mali","nan_Mauritania","nan_Mauritius","nan_Mongolia","nan_Mozambique","nan_Namibia","nan_Nepal",
"Aruba_Netherlands","Curacao_Netherlands","Sint Maarten_Netherlands","nan_Nicaragua","nan_Niger","nan_Papua New Guinea",
"nan_Saint Kitts and Nevis","nan_Saint Lucia","nan_Somalia","nan_Sudan","nan_Suriname","nan_Syria","nan_Tanzania",
"nan_Togo","Virgin Islands_US","Bermuda_United Kingdom","Cayman Islands_United Kingdom","Channel Islands_United Kingdom",
"Gibraltar_United Kingdom","Isle of Man_United Kingdom","nan_Zimbabwe"]
decay_1_5 =["nan_Dominican Republic","nan_Kazakhstan","nan_Tunisia",
"Alabama_US", "Alaska_US","Arizona_US","Colorado_US","Florida_US","Montana_US",
"Nebraska_US","Nevada_US","New Hampshire_US","New Mexico_US",
"Puerto Rico_US","nan_Ukraine","nan_Uzbekistan"]
linear_last_12=["nan_Uganda","nan_Zambia"]
stay_same=[ "nan_Antigua and Barbuda","nan_Diamond Princess","nan_Saint Vincent and the Grenadines","nan_Timor-Leste","Montserrat_United Kingdom"]
tr_frame=train_frame
target_confirmed_train=tr_frame[target_confirmed].values
print(" original shape of train is {} ".format(target_confirmed_train.shape))
target_fatalities_train=tr_frame[target_fatalities].values
features_train=tr_frame[names].values
standard_confirmed_train=tr_frame["ConfirmedCases"].values
standard_fatalities_train=tr_frame["Fatalities"].values
current_confirmed_train=tr_frame["ConfirmedCases"].values
features_cv=[]
name_cv=[]
standard_confirmed_cv=[]
standard_fatalities_cv=[]
names_=tr_frame["key"].values
training_horizon=int(features_train.shape[0]/len(unique_keys))
print("training horizon = ",training_horizon)
 for dd in range(training_horizon-1,features_train.shape[0],training_horizon):
features_cv.append(features_train[dd])
 name_cv.append(names_[dd])
 standard_confirmed_cv.append(standard_confirmed_train[dd])
 standard_fatalities_cv.append(standard_fatalities_train[dd])
 print(name_cv[-1], standard_confirmed_cv[-1], standard_fatalities_cv[-1])
 current_confirmed_train=[k for k in range(len(current_confirmed_train)) if current_confirmed_train[k]>0]
target_confirmed_train=target_confirmed_train[current_confirmed_train]
target_fatalities_train=target_fatalities_train[current_confirmed_train]
features_train=features_train[current_confirmed_train]
standard_confirmed_train=standard_confirmed_train[current_confirmed_train]
standard_fatalities_train=standard_fatalities_train[current_confirmed_train]
features_cv=np.array(features_cv)
 preds_confirmed_cv=np.zeros(( features_cv.shape[0],horizon))
preds_confirmed_standard_cv=np.zeros(( features_cv.shape[0],horizon))
preds_fatalities_cv=np.zeros(( features_cv.shape[0],horizon))
preds_fatalities_standard_cv=np.zeros(( features_cv.shape[0],horizon))
overal_rmsle_metric_confirmed=0.0
for j in range(preds_confirmed_cv.shape[1]):
this_target=target_confirmed_train[:,j]
index_positive=[k for k in range(len(this_target)) if this_target[k]!=-1]
this_features=features_train[index_positive]
this_target=this_target[index_positive]
this_weight=np.log(standard_confirmed_train[index_positive]+2.)
this_features_cv=features_cv
preds=bagged_set_train(this_features,this_target,this_weight, seed, 1,features_cv, xt=None,yt=None, output_name=None)
 preds_confirmed_cv[:,j]=preds
print(" modelling confirmed, case %d, original train %d, and after %d, original cv %d and after %d "%(
 j,target_confirmed_train.shape[0],this_target.shape[0],this_features_cv.shape[0],this_features_cv.shape[0]))
predictions=[]
for ii in range(preds_confirmed_cv.shape[0]):
current_prediction=standard_confirmed_cv[ii]
if current_prediction==0 :
current_prediction=0.1
this_preds=preds_confirmed_cv[ii].tolist()
name=name_cv[ii]
if name in decay_4_first_10_then_1:
this_preds=decay_4_first_10_then_1_f(this_preds)
 elif name in decay_2:
this_preds=decay_2_f(this_preds)
 elif name in decay_2_last_12_linear_inter:
this_preds=decay_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_5:
this_preds=decay_1_5_f(this_preds)
 elif name in linear_last_12:
this_preds=linear_last_12_f(this_preds)
 elif name in stay_same or "China" in name:
this_preds=stay_same_f(this_preds)
 for j in range(preds_confirmed_cv.shape[1]):
current_prediction*=max(1,this_preds[j])
 preds_confirmed_standard_cv[ii][j]=current_prediction
for j in range(preds_confirmed_cv.shape[1]):
this_target=target_fatalities_train[:,j]
index_positive=[k for k in range(len(this_target)) if this_target[k]!=-1]
this_features=features_train[index_positive]
this_target=this_target[index_positive]
this_weight=np.log(standard_confirmed_train[index_positive]+2.)
this_features_cv=features_cv
preds=bagged_set_train(this_features,this_target,this_weight, seed, 1,features_cv, xt=None,yt=None, output_name=None)
 preds_fatalities_cv[:,j]=preds
print(" modelling fatalities, case %d, original train %d, and after %d, original cv %d and after %d "%(
 j,target_confirmed_train.shape[0],this_target.shape[0],this_features_cv.shape[0],this_features_cv.shape[0]))
predictions=[]
for ii in range(preds_fatalities_cv.shape[0]):
current_prediction=standard_fatalities_cv[ii]
if current_prediction==0 and standard_confirmed_cv[ii]>400:
current_prediction=0.1
this_preds=preds_fatalities_cv[ii].tolist()
name=name_cv[ii]
if name in decay_4_first_10_then_1:
this_preds=decay_4_first_10_then_1_f(this_preds)
 elif name in decay_2:
this_preds=decay_2_f(this_preds)
 elif name in decay_2_last_12_linear_inter:
this_preds=decay_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_5:
this_preds=decay_1_5_f(this_preds)
 elif name in linear_last_12:
this_preds=linear_last_12_f(this_preds)
 elif name in stay_same or "China" in name:
this_preds=stay_same_f(this_preds)
 for j in range(preds_fatalities_cv.shape[1]):
if current_prediction==0 and preds_confirmed_standard_cv[ii][j]>400:
current_prediction=1.
 current_prediction*=max(1,this_preds[j])
 preds_fatalities_standard_cv[ii][j]=current_prediction
transforms = albumentations.Compose([
albumentations.Transpose(p=0.5),
albumentations.VerticalFlip(p=0.5),
albumentations.HorizontalFlip(p=0.5),
] )<categorify>
key_to_confirmed_rate={}
key_to_fatality_rate={}
key_to_confirmed={}
key_to_fatality={}
print(len(features_cv), len(name_cv),len(standard_confirmed_cv),len(standard_fatalities_cv))
print(preds_confirmed_cv.shape,preds_confirmed_standard_cv.shape,preds_fatalities_cv.shape,preds_fatalities_standard_cv.shape)
 for j in range(len(name_cv)) :
key_to_confirmed_rate[name_cv[j]]=preds_confirmed_cv[j,:].tolist()
key_to_fatality_rate[name_cv[j]]=preds_fatalities_cv[j,:].tolist()
key_to_confirmed[name_cv[j]] =preds_confirmed_standard_cv[j,:].tolist()
key_to_fatality[name_cv[j]]=preds_fatalities_standard_cv[j,:].tolist()
if os.path.exists(PRED_PATH):
predictions = []
for index, row in tqdm(df.iterrows() , total = df.shape[0]):
image_id = row['image_id']
img_path = PRED_PATH + image_id + '.tiff'
img = skimage.io.MultiImage(img_path)[1]
patches = tile(img)
 patches1 = patches.copy()
patches2 = patches.copy()
k = 0
while k < 42:
patches1[k, ] = transforms(image=patches1[k, ])['image']
patches2[k, ] = transforms(image=patches2[k, ])['image']
k += 1
image = np.stack([patches, patches1, patches2])
 image = image / 255.0
pred = model.predict(image)
 isup = np.round(np.mean(pred))
if isup < 0:
isup = 0
if isup > 5:
isup = 5
predictions.append(int(isup))
del patches, img
gc.collect() 
<save_to_csv>
train_new=train[["Date","ConfirmedCases","Fatalities","key","rate_ConfirmedCases","rate_Fatalities"]]
test_new=pd.merge(test,train_new, how="left", left_on=["key","Date"], right_on=["key","Date"] ).reset_index(drop=True)
 test_new
<define_variables><EOS> 
def fillin_columns(frame,key_column, original_name, training_horizon, test_horizon, unique_values, key_to_values):
keys=frame[key_column].values
original_values=frame[original_name].values.tolist()
print(len(keys), len(original_values), training_horizon ,test_horizon,len(key_to_values))
for j in range(unique_values):
current_index=(j *(training_horizon +test_horizon)) +training_horizon
current_key=keys[current_index]
values=key_to_values[current_key]
co=0
for g in range(current_index, current_index + test_horizon):
original_values[g]=values[co]
co+=1
frame[original_name]=original_values
all_days=int(test_new.shape[0]/len(unique_keys))
tr_horizon=all_days-horizon
print(all_days,tr_horizon, horizon)
 fillin_columns(test_new,"key", 'ConfirmedCases', tr_horizon, horizon, len(unique_keys), key_to_confirmed)
 fillin_columns(test_new,"key", 'Fatalities', tr_horizon, horizon, len(unique_keys), key_to_fatality)
 submission=test_new[["ForecastId","ConfirmedCases","Fatalities"]]
submission.to_csv("submission.csv", index=False)