Kaggle competition: COVID19 Global Forecasting (Week 2)
features = [48, 72, 'cp_dose', 'g-0', 'g-1', 'g-2', 'g-3', 'g-4', 'g-5', 'g-6', 'g-8', 'g-10', 'g-11', 'g-12', 'g-13', 'g-14', 'g-15', 'g-16', 'g-17', 'g-18', 'g-20', 'g-21', 'g-22', 'g-24', 'g-25', 'g-27', 'g-28', 'g-29', 'g-30', 'g-32', 'g-33', 'g-34', 'g-35', 'g-36', 'g-37', 'g-38', 'g-40', 'g-41', 'g-42', 'g-44', 'g-45', 'g-46', 'g-47', 'g-48', 'g-49', 'g-50', 'g-51', 'g-52', 'g-53', 'g-55', 'g-56', 'g-57', 'g-59', 'g-60', 'g-61', 'g-62', 'g-63', 'g-64', 'g-65', 'g-66', 'g-68', 'g-69', 'g-73', 'g-75', 'g-76', 'g-77', 'g-78', 'g-79', 'g-81', 'g-82', 'g-84', 'g-89', 'g-90', 'g-91', 'g-93', 'g-94', 'g-95', 'g-96', 'g-97', 'g-98', 'g-99', 'g-100', 'g-101', 'g-102', 'g-103', 'g-105', 'g-107', 'g-108', 'g-110', 'g-111', 'g-113', 'g-114', 'g-116', 'g-117', 'g-119', 'g-120', 'g-122', 'g-123', 'g-124', 'g-125', 'g-126', 'g-127', 'g-128', 'g-130', 'g-131', 'g-133', 'g-134', 'g-136', 'g-137', 'g-140', 'g-141', 'g-142', 'g-143', 'g-144', 'g-145', 'g-146', 'g-148', 'g-149', 'g-150', 'g-151', 'g-153', 'g-156', 'g-157', 'g-158', 'g-159', 'g-160', 'g-163', 'g-164', 'g-166', 'g-167', 'g-169', 'g-172', 'g-173', 'g-174', 'g-175', 'g-176', 'g-177', 'g-178', 'g-180', 'g-183', 'g-184', 'g-185', 'g-186', 'g-187', 'g-188', 'g-190', 'g-191', 'g-192', 'g-194', 'g-195', 'g-198', 'g-200', 'g-201', 'g-202', 'g-203', 'g-204', 'g-205', 'g-206', 'g-207', 'g-208', 'g-209', 'g-212', 'g-213', 'g-214', 'g-216', 'g-217', 'g-218', 'g-221', 'g-222', 'g-223', 'g-224', 'g-226', 'g-228', 'g-229', 'g-230', 'g-231', 'g-232', 'g-233', 'g-234', 'g-235', 'g-240', 'g-241', 'g-242', 'g-243', 'g-244', 'g-245', 'g-246', 'g-248', 'g-250', 'g-251', 'g-252', 'g-253', 'g-254', 'g-256', 'g-258', 'g-260', 'g-262', 'g-263', 'g-264', 'g-265', 'g-267', 'g-268', 'g-269', 'g-271', 'g-272', 'g-273', 'g-274', 'g-275', 'g-277', 'g-278', 'g-279', 'g-280', 'g-281', 'g-282', 'g-283', 'g-284', 'g-287', 'g-288', 'g-289', 'g-291', 'g-292', 'g-293', 'g-294', 'g-296', 'g-297', 'g-298', 'g-299', 'g-300', 'g-301', 'g-302', 'g-303', 'g-304', 'g-305', 'g-306', 'g-309', 'g-311', 'g-312', 'g-313', 'g-314', 'g-315', 'g-317', 'g-318', 'g-321', 'g-322', 'g-323', 'g-324', 'g-325', 'g-329', 'g-330', 'g-331', 'g-332', 'g-333', 'g-334', 'g-336', 'g-337', 'g-340', 'g-341', 'g-342', 'g-343', 'g-344', 'g-345', 'g-347', 'g-348', 'g-350', 'g-351', 'g-352', 'g-353', 'g-354', 'g-355', 'g-356', 'g-357', 'g-358', 'g-359', 'g-360', 'g-362', 'g-363', 'g-365', 'g-369', 'g-370', 'g-371', 'g-372', 'g-373', 'g-375', 'g-376', 'g-377', 'g-378', 'g-379', 'g-380', 'g-381', 'g-382', 'g-383', 'g-384', 'g-386', 'g-387', 'g-389', 'g-390', 'g-391', 'g-392', 'g-394', 'g-395', 'g-396', 'g-397', 'g-398', 'g-399', 'g-400', 'g-401', 'g-402', 'g-403', 'g-404', 'g-405', 'g-406', 'g-407', 'g-408', 'g-409', 'g-410', 'g-411', 'g-414', 'g-415', 'g-418', 'g-421', 'g-422', 'g-423', 'g-424', 'g-425', 'g-427', 'g-429', 'g-430', 'g-431', 'g-432', 'g-433', 'g-434', 'g-435', 'g-436', 'g-437', 'g-438', 'g-439', 'g-440', 'g-441', 'g-442', 'g-443', 'g-444', 'g-445', 'g-446', 'g-447', 'g-449', 'g-450', 'g-451', 'g-453', 'g-454', 'g-455', 'g-456', 'g-457', 'g-458', 'g-459', 'g-460', 'g-461', 'g-462', 'g-463', 'g-464', 'g-465', 'g-466', 'g-468', 'g-469', 'g-471', 'g-472', 'g-473', 'g-476', 'g-478', 'g-479', 'g-480', 'g-481', 'g-482', 'g-483', 'g-485', 'g-486', 'g-487', 'g-489', 'g-490', 'g-491', 'g-492', 'g-493', 'g-494', 'g-495', 'g-497', 'g-499', 'g-500', 'g-503', 'g-505', 'g-506', 'g-507', 'g-509', 'g-511', 'g-512', 'g-513', 'g-515', 'g-516', 'g-518', 'g-519', 'g-520', 'g-521', 'g-522', 'g-524', 'g-525', 'g-526', 'g-527', 'g-528', 'g-530', 'g-533', 'g-534', 'g-536', 'g-537', 'g-538', 'g-539', 'g-540', 'g-541', 'g-543', 'g-544', 'g-547', 'g-550', 'g-551', 'g-552', 'g-553', 'g-554', 'g-555', 'g-556', 'g-557', 'g-558', 'g-559', 'g-560', 'g-561', 'g-562', 'g-563', 'g-564', 'g-565', 'g-566', 'g-567', 'g-569', 'g-570', 'g-571', 'g-572', 'g-573', 'g-575', 'g-578', 'g-581', 'g-583', 'g-584', 'g-585', 'g-586', 'g-587', 'g-588', 'g-589', 'g-590', 'g-594', 'g-596', 'g-597', 'g-598', 'g-599', 'g-600', 'g-601', 'g-602', 'g-603', 'g-604', 'g-605', 'g-606', 'g-608', 'g-610', 'g-611', 'g-612', 'g-613', 'g-614', 'g-615', 'g-616', 'g-617', 'g-618', 'g-619', 'g-620', 'g-621', 'g-623', 'g-624', 'g-625', 'g-626', 'g-627', 'g-628', 'g-630', 'g-631', 'g-633', 'g-634', 'g-635', 'g-636', 'g-638', 'g-639', 'g-640', 'g-643', 'g-645', 'g-648', 'g-649', 'g-651', 'g-655', 'g-656', 'g-657', 'g-658', 'g-659', 'g-661', 'g-663', 'g-664', 'g-665', 'g-666', 'g-667', 'g-668', 'g-669', 'g-670', 'g-671', 'g-672', 'g-674', 'g-676', 'g-677', 'g-678', 'g-679', 'g-680', 'g-681', 'g-682', 'g-683', 'g-688', 'g-689', 'g-690', 'g-691', 'g-692', 'g-694', 'g-696', 'g-697', 'g-698', 'g-699', 'g-701', 'g-703', 'g-704', 'g-705', 'g-707', 'g-709', 'g-710', 'g-711', 'g-712', 'g-713', 'g-714', 'g-715', 'g-716', 'g-718', 'g-719', 'g-720', 'g-721', 'g-722', 'g-723', 'g-724', 'g-725', 'g-726', 'g-727', 'g-729', 'g-730', 'g-731', 'g-732', 'g-733', 'g-734', 'g-735', 'g-736', 'g-737', 'g-741', 'g-742', 'g-745', 'g-747', 'g-752', 'g-753', 'g-754', 'g-755', 'g-757', 'g-758', 'g-759', 'g-761', 'g-763', 'g-764', 'g-765', 'g-766', 'g-767', 'g-768', 'g-769', 'g-770', 'g-771', 'c-1', 'c-3', 'c-4', 'c-7', 'c-8', 'c-9', 'c-12', 'c-13', 'c-14', 'c-15', 'c-16', 'c-17', 'c-18', 'c-21', 'c-22', 'c-23', 'c-24', 'c-25', 'c-26', 'c-27', 'c-29', 'c-30', 'c-31', 'c-34', 'c-35', 'c-37', 'c-38', 'c-39', 'c-40', 'c-41', 'c-42', 'c-44', 'c-46', 'c-47', 'c-48', 'c-49', 'c-50', 'c-51', 'c-54', 'c-56', 'c-57', 'c-58', 'c-59', 'c-61', 'c-62', 'c-64', 'c-65', 'c-66', 'c-67', 'c-69', 'c-73', 'c-76', 'c-77', 'c-79', 'c-81', 'c-82', 'c-86', 'c-87', 'c-89', 'c-91', 'c-93', 'c-94', 'c-95', 'c-98', 'c-99']<split>
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5
def fix_target(frame, key, target, new_target_name="target"):
corrections = 0
group_keys = frame[ key].values.tolist()
target = frame[target].values.tolist()
for i in range(1, len(group_keys)- 1):
previous_group = group_keys[i - 1]
current_group = group_keys[i]
previous_value = target[i - 1]
current_value = target[i]
if current_group == previous_group:
if current_value<previous_value:
current_value=previous_value
target[i] =current_value
target[i] =max(0,target[i])
 frame[new_target_name] = np.array(target)
 def rate(frame, key, target, new_target_name="rate"):
corrections = 0
group_keys = frame[ key].values.tolist()
target = frame[target].values.tolist()
rate=[1.0 for k in range(len(target)) ]
for i in range(1, len(group_keys)- 1):
previous_group = group_keys[i - 1]
current_group = group_keys[i]
previous_value = target[i - 1]
current_value = target[i]
if current_group == previous_group:
if previous_value!=0.0:
rate[i]=current_value/previous_value
rate[i] =max(1,rate[i])
 frame[new_target_name] = np.array(rate)
 def get_data_by_key(dataframe, key, key_value, fields=None):
mini_frame=dataframe[dataframe[key]==key_value]
if not fields is None:
mini_frame=mini_frame[fields].values
return mini_frame
directory="/kaggle/input/covid19-global-forecasting-week-2/"
model_directory="/kaggle/input/model-dir/model"
train=pd.read_csv(directory + "train.csv", parse_dates=["Date"] , engine="python")
 test=pd.read_csv(directory + "test.csv", parse_dates=["Date"], engine="python")
 train["key"]=train[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 test["key"]=test[["Province_State","Country_Region"]].apply(lambda row: str(row[0])+ "_" + str(row[1]),axis=1)
 max_train_date=train["Date"].max()
max_test_date=test["Date"].max()
horizon=(max_test_date-max_train_date ).days
print("horizon", int(horizon))
target1="ConfirmedCases"
target2="Fatalities"
key="key"
train = train_data.loc[:,features]
test = test_data.loc[:,features]<define_search_model>
fix_target(train, key, target1, new_target_name=target1)
 rate(train, key, target1, new_target_name="rate_" +target1)
 rate(train, key, target2, new_target_name="rate_" +target2)
 unique_keys=train[key].unique()
print(len(unique_keys))
train
def fe_cluster(train, test, n_clusters_g = 35, n_clusters_c = 5, SEED = 123):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_{kind}'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_{kind}'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
a ,b=fe_cluster(train_features,test_features)
 cols = [col for col in a.columns if col.startswith('clusters')]
train = pd.concat([train,a.loc[train_features[train_features.cp_type=="trt_cp"].index,cols]],axis=1)
 test = pd.concat([test,b.loc[:,cols]],axis=1 )<feature_engineering>
def get_lags(rate_array, current_index, size=20):
lag_confirmed_rate=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
lag_confirmed_rate[j]=rate_array[current_index-j]
else :
break
return lag_confirmed_rate
def days_ago_thresold_hit(full_array, indx, thresold):
days_ago_confirmed_count_10=-1
if full_array[indx]>thresold:
for j in range(indx,-1,-1):
entered=False
if full_array[j]<=thresold:
days_ago_confirmed_count_10=abs(j-indx)
 entered=True
break
if entered==False:
days_ago_confirmed_count_10=100
return days_ago_confirmed_count_10
def ewma_vectorized(data, alpha):
sums=sum([(alpha**(k+1)) *data[k] for k in range(len(data)) ])
 counts=sum([(alpha**(k+1)) for k in range(len(data)) ])
 return sums/counts
def generate_ma_std_window(rate_array, current_index, size=20, window=3):
ma_rate_confirmed=[-1 for k in range(size)]
std_rate_confirmed=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
ma_rate_confirmed[j]=np.mean(rate_array[max(0,current_index-j-window+1):current_index-j+1])
 std_rate_confirmed[j]=np.std(rate_array[max(0,current_index-j-window+1):current_index-j+1])
 else :
break
return ma_rate_confirmed, std_rate_confirmed
def generate_ewma_window(rate_array, current_index, size=20, window=3, alpha=0.05):
ewma_rate_confirmed=[-1 for k in range(size)]
for j in range(0, size):
if current_index-j>=0:
ewma_rate_confirmed[j]=ewma_vectorized(rate_array[max(0,current_index-j-window+1):current_index-j+1, ], alpha)
 else :
break
return ewma_rate_confirmed
def get_target(rate_col, indx, horizon=33, average=3, use_hard_rule=False):
target_values=[-1 for k in range(horizon)]
cou=0
for j in range(indx+1, indx+1+horizon):
if j<len(rate_col):
if average==1:
target_values[cou]=rate_col[j]
else :
if use_hard_rule and j +average <=len(rate_col):
target_values[cou]=np.mean(rate_col[j:j +average])
 else :
target_values[cou]=np.mean(rate_col[j:min(len(rate_col),j +average)])
 cou+=1
else :
break
return target_values
def dereive_features(frame, confirmed, fatalities, rate_confirmed, rate_fatalities,
horizon ,size=20, windows=[3,7], days_back_confimed=[1,10,100], days_back_fatalities=[1,2,10]):
targets=[]
names=["lag_confirmed_rate" + str(k+1)for k in range(size)]
for day in days_back_confimed:
names+=["days_ago_confirmed_count_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["std" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["ewma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["lag_fatalities_rate" + str(k+1)for k in range(size)]
for day in days_back_fatalities:
names+=["days_ago_fatalitiescount_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["std" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["ewma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["confirmed_level"]
names+=["fatalities_level"]
names+=["confirmed_plus" + str(k+1)for k in range(horizon)]
names+=["fatalities_plus" + str(k+1)for k in range(horizon)]
features=[]
for i in range(len(confirmed)) :
row_features=[]
lag_confirmed_rate=get_lags(rate_confirmed, i, size=size)
 row_features+=lag_confirmed_rate
for day in days_back_confimed:
days_ago_confirmed_count_10=days_ago_thresold_hit(confirmed, i, day)
 row_features+=[days_ago_confirmed_count_10]
for window in windows:
ma3_rate_confirmed,std3_rate_confirmed= generate_ma_std_window(rate_confirmed, i, size=size, window=window)
 row_features+= ma3_rate_confirmed
row_features+= std3_rate_confirmed
ewma3_rate_confirmed=generate_ewma_window(rate_confirmed, i, size=size, window=window, alpha=0.05)
 row_features+= ewma3_rate_confirmed
lag_fatalities_rate=get_lags(rate_fatalities, i, size=size)
 row_features+=lag_fatalities_rate
for day in days_back_fatalities:
days_ago_fatalitiescount_2=days_ago_thresold_hit(fatalities, i, day)
 row_features+=[days_ago_fatalitiescount_2]
for window in windows:
ma3_rate_fatalities,std3_rate_fatalities= generate_ma_std_window(rate_fatalities, i, size=size, window=window)
 row_features+= ma3_rate_fatalities
row_features+= std3_rate_fatalities
ewma3_rate_fatalities=generate_ewma_window(rate_fatalities, i, size=size, window=window, alpha=0.05)
 row_features+= ewma3_rate_fatalities
confirmed_level=0

 confirmed_level= confirmed[i]
row_features+=[confirmed_level]
fatalities_is_level=0

 fatalities_is_level= fatalities[i]
row_features+=[fatalities_is_level]
confirmed_plus=get_target(rate_confirmed, i, horizon=horizon)
 row_features+= confirmed_plus
fatalities_plus=get_target(rate_fatalities, i, horizon=horizon)
 row_features+= fatalities_plus
features.append(row_features)
 new_frame=pd.DataFrame(data=features, columns=names ).reset_index(drop=True)
 frame=frame.reset_index(drop=True)
 frame=pd.concat([frame, new_frame], axis=1)
 return frame
def feature_engineering_for_single_key(frame, group, key, horizon=33, size=20, windows=[3,7],
days_back_confimed=[1,10,100], days_back_fatalities=[1,2,10]):
mini_frame=get_data_by_key(frame, group, key, fields=None)
 mini_frame_with_features=dereive_features(mini_frame, mini_frame["ConfirmedCases"].values,
mini_frame["Fatalities"].values, mini_frame["rate_ConfirmedCases"].values,
mini_frame["rate_Fatalities"].values, horizon ,size=size, windows=windows,
days_back_confimed=days_back_confimed, days_back_fatalities=days_back_fatalities)
 return mini_frame_with_features
def fe_stats(train, test):
train_ = train.copy()
test_ = test.copy()
features_g = list(train_features.columns[4:776])
 features_c = list(train_features.columns[776:876])
 for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 return train.iloc[:,-15:], test.iloc[:,-15:]
a,b = fe_stats(train_features,test_features)
<concatenate>
def predict(xtest,input_name=None):
baggedpred=np.array([ 0.0 for d in range(0, xtest.shape[0])])
 model= joblib.load(input_name)
 preds=model.predict(xtest)
 baggedpred+=preds
return baggedpred
train = pd.concat(( train,a.loc[train_features[train_features.cp_type=="trt_cp"].index,:]),axis=1)
 test = pd.concat(( test,b),axis=1 )<normalization>
names=["lag_confirmed_rate" + str(k+1)for k in range(size)]
for day in days_back_confimed:
names+=["days_ago_confirmed_count_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["std" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["ewma" + str(window)+ "_rate_confirmed" + str(k+1)for k in range(size)]
names+=["lag_fatalities_rate" + str(k+1)for k in range(size)]
for day in days_back_fatalities:
names+=["days_ago_fatalitiescount_" + str(day)]
for window in windows:
names+=["ma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["std" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["ewma" + str(window)+ "_rate_fatalities" + str(k+1)for k in range(size)]
names+=["confirmed_level"]
names+=["fatalities_level"]
n_comps = 600
pca = PCA(n_components = n_comps)
 total_data = train_features.append(test_features)
 total_data = total_data.loc[:,GENES]
pca.fit(total_data)
 train2 = pca.transform(train_features.loc[train_features[train_features.cp_type=="trt_cp"].index,GENES])
 test2 = pca.transform(test_features.loc[:,GENES])
 train2 = pd.DataFrame(train2,columns=[f'pca-g_{i}' for i in range(train2.shape[1])])
 test2 = pd.DataFrame(test2,columns=[f'pca-g_{i}' for i in range(test2.shape[1])])
 total = train2.append(test2)
 vt = VarianceThreshold(0.8)
 vt.fit(total)
 train2 = vt.transform(train2)
 test2 = vt.transform(test2)
 train2 = pd.DataFrame(train2,columns=[f'pca-g_{i}' for i in range(train2.shape[1])])
 test2 = pd.DataFrame(test2,columns=[f'pca-g_{i}' for i in range(test2.shape[1])])
 train = pd.concat(( train.reset_index(drop=True),train2),axis=1)
 test = pd.concat(( test,test2),axis=1 )<normalization>
def decay_4_first_10_then_1_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
if j<10:
arr[j]=1.+(max(1,array[j])-1.) /4.
 else :
arr[j]=1.
 return arr
def decay_2_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /2.
 return arr
def acceleratorx2_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) *2.
 return arr
def decay_1_5_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /1.5
return arr
def stay_same_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.
 return arr
def decay_2_last_12_linear_inter_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=1.+(max(1,array[j])-1.) /2.
 arr12=(max(1,arr[-12])-1.) /12.
 for j in range(0, 12):
arr[len(arr)-12 +j]= max(1, 1 +(( arr12*12)-(j+1)*arr12))
return arr
def linear_last_12_f(array):
arr=[1.0 for k in range(len(array)) ]
for j in range(len(array)) :
arr[j]=max(1,array[j])
 arr12=(max(1,arr[-12])-1.) /12.
 for j in range(0, 12):
arr[len(arr)-12 +j]= max(1, 1 +(( arr12*12)-(j+1)*arr12))
return arr
decay_4_first_10_then_1 =["Beijing_China","Fujian_China","Guangdong_China", "Hong Kong_China",
"Inner Mongolia_China","Jiangsu_China","Liaoning_China","Macau_China","Shandong_China","Tianjin_China",
"Yunnan_China","Zhejiang_China","Northern Territory_Australia",
"nan_Belize","nan_Benin","nan_Bhutan","nan_Seychelles","nan_Cabo Verde"]
decay_2 =["Shanghai_China" , "nan_Afghanistan","nan_Andorra","Australian Capital Territory_Australia",
"South Australia_Australia","Tasmania_Australia","nan_Bahrain","nan_Belarus"
"nan_Belgium","nan_Bolivia","Manitoba_Canada","New Brunswick_Canada","Newfoundland and Labrador_Canada",
"Saskatchewan_Canada","nan_Central African Republic","nan_Congo(Kinshasa)","nan_Cote d'Ivoire","Mayotte_France","nan_Ukraine"]
stay_same=["China", "nan_Antigua and Barbuda","nan_Diamond Princess","nan_Saint Vincent and the Grenadines",
"nan_Timor-Leste","Montserrat_United Kingdom"]
decay_2_last_12_linear_inter =["nan_Angola" , "nan_Barbados" ,"Prince Edward Island_Canada","nan_Chad",
"nan_Congo(Brazzaville)","Greenland_Denmark","nan_Djibouti","nan_Dominica","nan_El Salvador",
"nan_Eritrea","nan_Eswatini","nan_Fiji","French Guiana_France","French Polynesia_France","New Caledonia_France",
"Saint Barthelemy_France","St Martin_France","nan_Gabon","nan_Gambia","nan_Grenada","nan_Guinea-Bissau",
"nan_Guyana","nan_Haiti","nan_Holy See","nan_Kyrgyzstan","nan_Laos","nan_Libya","nan_Madagascar",
"nan_Maldives","nan_Mali","nan_Mauritania","nan_Mauritius","nan_Mozambique","nan_Nepal",
"Aruba_Netherlands","Curacao_Netherlands","Sint Maarten_Netherlands","nan_Nicaragua","nan_Niger","nan_Papua New Guinea",
"nan_Saint Kitts and Nevis","nan_Saint Lucia","nan_Somalia","nan_Sudan","nan_Suriname","nan_Syria","nan_Tanzania",
"nan_Togo","Virgin Islands_US","Bermuda_United Kingdom","Cayman Islands_United Kingdom","Channel Islands_United Kingdom",
"Gibraltar_United Kingdom","Isle of Man_United Kingdom","nan_Zimbabwe","nan_Bahamas","nan_Zambia"]
acceleratorx2=["nan_Kenya","nan_Moldova"]
decay_1_5 =["nan_Kazakhstan","nan_Tunisia", "Alabama_US", "Alaska_US",
	"Arizona_US","Colorado_US","Florida_US","Montana_US","Nebraska_US","Nevada_US","New Hampshire_US","New Mexico_US",
	"Puerto Rico_US","nan_Uzbekistan","nan_Azerbaijan","nan_Bangladesh","nan_Bosnia and Herzegovina",
	"nan_Cameroon","nan_Cuba","nan_Guatemala","nan_Jamaica","nan_Morocco","nan_New Zealand","nan_Philippines","nan_Romania",
	"nan_Trinidad and Tobago"]
linear_last_12=["nan_Uganda","nan_Equatorial Guinea","nan_Guinea","nan_Honduras","nan_Liberia","nan_Mongolia","nan_Namibia"]
stay_same=[ "nan_Antigua and Barbuda","nan_Diamond Princess","nan_Saint Vincent and the Grenadines","nan_Timor-Leste","Montserrat_United Kingdom"]
tr_frame=train_frame
features_train=tr_frame[names].values
standard_confirmed_train=tr_frame["ConfirmedCases"].values
standard_fatalities_train=tr_frame["Fatalities"].values
current_confirmed_train=tr_frame["ConfirmedCases"].values
features_cv=[]
name_cv=[]
standard_confirmed_cv=[]
standard_fatalities_cv=[]
names_=tr_frame["key"].values
training_horizon=int(features_train.shape[0]/len(unique_keys))
print("training horizon = ",training_horizon)
 for dd in range(training_horizon-1,features_train.shape[0],training_horizon):
features_cv.append(features_train[dd])
 name_cv.append(names_[dd])
 standard_confirmed_cv.append(standard_confirmed_train[dd])
 standard_fatalities_cv.append(standard_fatalities_train[dd])
 print(name_cv[-1], standard_confirmed_cv[-1], standard_fatalities_cv[-1])
 features_cv=np.array(features_cv)
 preds_confirmed_cv=np.zeros(( features_cv.shape[0],horizon))
preds_confirmed_standard_cv=np.zeros(( features_cv.shape[0],horizon))
preds_fatalities_cv=np.zeros(( features_cv.shape[0],horizon))
preds_fatalities_standard_cv=np.zeros(( features_cv.shape[0],horizon))
overal_rmsle_metric_confirmed=0.0
for j in range(preds_confirmed_cv.shape[1]):
this_features_cv=features_cv
preds=predict(features_cv, input_name=model_directory +"confirmed"+ str(j))
preds_confirmed_cv[:,j]=preds
print(" modelling confirmed, case %d, , original cv %d and after %d "%(j,this_features_cv.shape[0],this_features_cv.shape[0]))
predictions=[]
for ii in range(preds_confirmed_cv.shape[0]):
current_prediction=standard_confirmed_cv[ii]
if current_prediction==0 :
current_prediction=0.1
this_preds=preds_confirmed_cv[ii].tolist()
name=name_cv[ii]
if name in decay_4_first_10_then_1:
this_preds=decay_4_first_10_then_1_f(this_preds)
 elif name in decay_2:
this_preds=decay_2_f(this_preds)
 elif name in decay_2_last_12_linear_inter:
this_preds=decay_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_5:
this_preds=decay_1_5_f(this_preds)
 elif name in linear_last_12:
this_preds=linear_last_12_f(this_preds)
 elif name in acceleratorx2:
this_preds=acceleratorx2_f(this_preds)
 elif name in stay_same or "China" in name:
this_preds=stay_same_f(this_preds)
 for j in range(preds_confirmed_cv.shape[1]):
current_prediction*=max(1,this_preds[j])
 preds_confirmed_standard_cv[ii][j]=current_prediction
for j in range(preds_confirmed_cv.shape[1]):
this_features_cv=features_cv
preds=predict(features_cv, input_name=model_directory +"fatal"+ str(j))
preds_fatalities_cv[:,j]=preds
print(" modelling fatalities, case %d, original cv %d and after %d "%(j,this_features_cv.shape[0],this_features_cv.shape[0]))
predictions=[]
for ii in range(preds_fatalities_cv.shape[0]):
current_prediction=standard_fatalities_cv[ii]
if current_prediction==0 and standard_confirmed_cv[ii]>400:
current_prediction=0.1
this_preds=preds_fatalities_cv[ii].tolist()
name=name_cv[ii]
if name in decay_4_first_10_then_1:
this_preds=decay_4_first_10_then_1_f(this_preds)
 elif name in decay_2:
this_preds=decay_2_f(this_preds)
 elif name in decay_2_last_12_linear_inter:
this_preds=decay_2_last_12_linear_inter_f(this_preds)
 elif name in decay_1_5:
this_preds=decay_1_5_f(this_preds)
 elif name in linear_last_12:
this_preds=linear_last_12_f(this_preds)
 elif name in acceleratorx2:
this_preds=acceleratorx2_f(this_preds)
 elif name in stay_same or "China" in name:
this_preds=stay_same_f(this_preds)
 for j in range(preds_fatalities_cv.shape[1]):
if current_prediction==0 and preds_confirmed_standard_cv[ii][j]>400:
current_prediction=1.
 current_prediction*=max(1,this_preds[j])
 preds_fatalities_standard_cv[ii][j]=current_prediction
n_comps = 50
pca = PCA(n_components = n_comps)
 total_data = train_features.append(test_features)
 total_data = total_data.loc[:,CELLS]
pca.fit(total_data)
 train2 = pca.transform(train_features.loc[train_features[train_features.cp_type=="trt_cp"].index,CELLS])
 test2 = pca.transform(test_features.loc[:,CELLS])
 train2 = pd.DataFrame(train2,columns=[f'pca-c_{i}' for i in range(train2.shape[1])])
 test2 = pd.DataFrame(test2,columns=[f'pca-c_{i}' for i in range(test2.shape[1])])
 total = train2.append(test2)
 vt = VarianceThreshold(0.8)
 vt.fit(total)
 train2 = vt.transform(train2)
 test2 = vt.transform(test2)
 train2 = pd.DataFrame(train2,columns=[f'pca-c_{i}' for i in range(train2.shape[1])])
 test2 = pd.DataFrame(test2,columns=[f'pca-c_{i}' for i in range(test2.shape[1])])
 train = pd.concat(( train.reset_index(drop=True),train2),axis=1)
 test = pd.concat(( test,test2),axis=1 )<load_from_csv>
key_to_confirmed_rate={}
key_to_fatality_rate={}
key_to_confirmed={}
key_to_fatality={}
print(len(features_cv), len(name_cv),len(standard_confirmed_cv),len(standard_fatalities_cv))
print(preds_confirmed_cv.shape,preds_confirmed_standard_cv.shape,preds_fatalities_cv.shape,preds_fatalities_standard_cv.shape)
 for j in range(len(name_cv)) :
key_to_confirmed_rate[name_cv[j]]=preds_confirmed_cv[j,:].tolist()
key_to_fatality_rate[name_cv[j]]=preds_fatalities_cv[j,:].tolist()
key_to_confirmed[name_cv[j]] =preds_confirmed_standard_cv[j,:].tolist()
key_to_fatality[name_cv[j]]=preds_fatalities_standard_cv[j,:].tolist()
Y_transfer = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 Y_transfer = Y_transfer.loc[train_features[train_features.cp_type=="trt_cp"].index,:].drop('sig_id',axis=1 )<split>
train_new=train[["Date","ConfirmedCases","Fatalities","key","rate_ConfirmedCases","rate_Fatalities"]]
test_new=pd.merge(test,train_new, how="left", left_on=["key","Date"], right_on=["key","Date"] ).reset_index(drop=True)
 test_new
X_train, X_val,Y_train,Y_val = train_test_split(train,Y_transfer,test_size=0.2,random_state=101 )<choose_model_class>
def fillin_columns(frame,key_column, original_name, training_horizon, test_horizon, unique_values, key_to_values):
keys=frame[key_column].values
original_values=frame[original_name].values.tolist()
print(len(keys), len(original_values), training_horizon ,test_horizon,len(key_to_values))
for j in range(unique_values):
current_index=(j *(training_horizon +test_horizon)) +training_horizon
current_key=keys[current_index]
values=key_to_values[current_key]
co=0
for g in range(current_index, current_index + test_horizon):
original_values[g]=values[co]
co+=1
frame[original_name]=original_values
all_days=int(test_new.shape[0]/len(unique_keys))
tr_horizon=all_days-horizon
print(all_days,tr_horizon, horizon)
 fillin_columns(test_new,"key", 'ConfirmedCases', tr_horizon, horizon, len(unique_keys), key_to_confirmed)
 fillin_columns(test_new,"key", 'Fatalities', tr_horizon, horizon, len(unique_keys), key_to_fatality)
 submission=test_new[["ForecastId","ConfirmedCases","Fatalities"]]
submission.to_csv("submission.csv", index=False)