Kaggle competition: COVID19 Global Forecasting (Week 2)
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )<define_variables>
path = '/kaggle/input/covid19-global-forecasting-week-2/'
pd.set_option('display.max_rows', None)
 train_df = pd.read_csv(path+'train.csv')
 test_df = pd.read_csv(path+'test.csv')
 submission_df = pd.read_csv(path+'submission.csv' )
data_path = ".. /input/lish-moa/"
no_ctl = True
scale = "rankgauss"
variance_threshould = 0.7
decompo = "PCA"
ncompo_genes = 80
ncompo_cells = 10
encoding = "dummy"<load_from_csv>
test_df['Date']= pd.to_datetime(test_df['Date'])
 train_df['Date']= pd.to_datetime(train_df['Date'] )
train = pd.read_csv(data_path + "train_features.csv")
 targets = pd.read_csv(data_path + "train_targets_scored.csv")
 test = pd.read_csv(data_path + "test_features.csv")
 submission = pd.read_csv(data_path + "sample_submission.csv" )<filter>
null_vals = pd.DataFrame(round(100 *(train_df.isnull().sum() / len(train_df.index)) , 3))
null_vals.columns = ['% null values']
null_vals
if no_ctl:
print(b_, "not_ctl")
 train = train[train["cp_type"] != "ctl_vehicle"]
test = test[test["cp_type"] != "ctl_vehicle"]
targets = targets.iloc[train.index]
train.reset_index(drop = True, inplace = True)
 test.reset_index(drop = True, inplace = True)
 targets.reset_index(drop = True, inplace = True )<define_variables>
null_vals = pd.DataFrame(round(100 *(test_df.isnull().sum() / len(test_df.index)) , 3))
null_vals.columns = ['% null values']
null_vals
GENES = [col for col in train.columns if col.startswith("g-")]
CELLS = [col for col in train.columns if col.startswith("c-")]<concatenate>
train_df['Province_State'].fillna('NO_STATE', inplace=True)
 test_df['Province_State'].fillna('NO_STATE', inplace=True )
data_all = pd.concat([train, test], ignore_index = True)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]
mask =(data_all[cols_numeric].var() >= variance_threshould ).values
tmp = data_all[cols_numeric].loc[:, mask]
data_all = pd.concat([data_all[["sig_id", "cp_type", "cp_time", "cp_dose"]], tmp], axis = 1)
 cols_numeric = [feat for feat in list(data_all.columns)if feat not in ["sig_id", "cp_type", "cp_time", "cp_dose"]]<normalization>
train_grouped_df = train_df.groupby(['Country_Region', 'Province_State'] )
def scale_minmax(col):
return(col - col.min())/(col.max() - col.min())
 def scale_norm(col):
return(col - col.mean())/ col.std()
if scale == "boxcox":
print(b_, "boxcox")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis = 0)
 trans = []
for feat in cols_numeric:
trans_var, lambda_var = stats.boxcox(data_all[feat].dropna() + 1)
 trans.append(scale_minmax(trans_var))
data_all[cols_numeric] = np.asarray(trans ).T
elif scale == "norm":
print(b_, "norm")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_norm, axis = 0)
 elif scale == "minmax":
print(b_, "minmax")
 data_all[cols_numeric] = data_all[cols_numeric].apply(scale_minmax, axis = 0)
 elif scale == "rankgauss":
print(b_, "Rank Gauss")
 scaler = GaussRankScaler()
data_all[cols_numeric] = scaler.fit_transform(data_all[cols_numeric])
 else:
pass<create_dataframe>
cluster0 = ['Botswana', 'Fiji', 'Gabon', 'Libya', 'Seychelles', 'Suriname', 'Mongolia', 'El Salvador', 'Barbados', 'Jamaica', 'Paraguay', 'Trinidad and Tobago','Montenegro', 'Georgia', 'Mauritius', 'Venezuela', 'Belarus', 'Malta', 'Oman', 'Albania', 'Cyprus', 'Kuwait', 'Jordan', 'Azerbaijan','Kazakhstan', 'Uruguay', 'Costa Rica', 'Bulgaria', 'Tunisia', 'Bosnia and Herzegovina', 'Latvia', 'Lebanon', 'Hungary', 'Armenia','Lithuania', 'Bahrain', 'Ukraine', 'Algeria', 'United Arab Emirates', 'Qatar', 'Estonia', 'Slovenia', 'Serbia', 'Colombia', 'Argentina','Dominican Republic', 'Peru', 'Mexico', 'Panama', 'Greece', 'South Africa', 'Indonesia', 'Saudi Arabia', 'Thailand', 'Ecuador', 'Poland','Romania', 'Malaysia', 'Brazil', 'Israel', 'Portugal', 'Turkey']
cluster0_confirmed_cap = 1500
cluster0_fatality_cap = 750
cluster1 = ['Italy', 'Spain']
cluster1_confirmed_cap = 50
cluster1_fatality_cap = 25
cluster2 = ['Iceland', 'Luxembourg', 'Switzerland']
cluster2_confirmed_cap = 500
cluster2_fatality_cap = 250
cluster3 = ['Angola', 'Bangladesh', 'Belize', 'Benin', 'Bhutan', 'Burkina Faso', 'Cambodia', 'Cameroon', 'Chad', 'Egypt', 'Ethiopia', 'Ghana', 'Guatemala', 'Guinea', 'Guyana', 'Haiti', 'Honduras', 'Iraq', 'Kenya', 'Liberia', 'Madagascar', 'Mali', 'Mauritania', 'Morocco', 'Mozambique', 'Namibia', 'Nepal','Nicaragua', 'Niger', 'Nigeria', 'Pakistan', 'Papua New Guinea', 'Philippines', 'Rwanda', 'Senegal', 'Sri Lanka', 'Sudan', 'Togo', 'Uganda','Zambia', 'Zimbabwe']
cluster3_confirmed_cap = 300
cluster3_fatality_cap = 150
cluster4 = ['China']
cluster4_confirmed_cap = 1.5
cluster4_fatality_cap = 1.35
cluster5 = ['Hong Kong', 'Singapore']
cluster5_confirmed_cap = 40
cluster5_fatality_cap = 28
cluster6 = ['United States']
cluster6_confirmed_cap = 200
cluster6_fatality_cap = 140
cluster6 = ['United States']
cluster6_confirmed_cap = 500
cluster6_fatality_cap = 350
cluster7 = ['Australia', 'Austria', 'Belgium', 'Canada', 'Chile', 'Denmark', 'Finland', 'Ireland', 'Japan', 'Netherlands', 'New Zealand', 'Norway', 'South Korea','Sweden', 'United Kingdom']
cluster7_confirmed_cap = 100
cluster7_fatality_cap = 70
cluster8 = ['France', 'Germany', 'Iran']
cluster8_confirmed_cap = 50
cluster8_fatality_cap = 35
cluster9 = ['India']
cluster9_confirmed_cap = 30
cluster9_fatality_cap = 21
if decompo == "PCA":
print(b_, "PCA")
 GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
pca_genes = PCA(n_components = ncompo_genes,
random_state = seed ).fit_transform(data_all[GENES])
 pca_cells = PCA(n_components = ncompo_cells,
random_state = seed ).fit_transform(data_all[CELLS])
 pca_genes = pd.DataFrame(pca_genes, columns = [f"pca_g-{i}" for i in range(ncompo_genes)])
 pca_cells = pd.DataFrame(pca_cells, columns = [f"pca_c-{i}" for i in range(ncompo_cells)])
 data_all = pd.concat([data_all, pca_genes, pca_cells], axis = 1)
 else:
pass<categorify>
predictions_dataset = pd.DataFrame([])
 for key, state_df in train_grouped_df:
print('Key:', key)
 train_start_date = state_df['Date'].iloc[0]
state_df['days_elapsed'] =(state_df['Date'] - train_start_date ).dt.days
train_state_df = state_df.drop(['Id' , 'Country_Region', 'Province_State'], axis = 1)
 y_train_case_count = train_state_df['ConfirmedCases']
y_train_fatalities = train_state_df['Fatalities']
x_train_state_case_df = train_state_df.drop(['Fatalities'], axis = 1)
 x_train_state_fatalities_df = train_state_df.drop('ConfirmedCases', axis = 1)
 test_state_df = pd.DataFrame([])
 test_state_df = test_df[(test_df['Country_Region'] == key[0])&(test_df['Province_State'] == key[1])]
test_state_df['days_elapsed'] =(test_state_df['Date'] - train_start_date ).dt.days
x_pred_state_df = test_state_df.drop(['ForecastId' ,'Province_State', 'Country_Region'], axis = 1)
 prophet_conf = Prophet(growth='logistic')
 prophet_fatalities = Prophet(growth='logistic')
 train_dataset = pd.DataFrame()
test_dataset = pd.DataFrame()
train_dataset['y'] = x_train_state_case_df['ConfirmedCases']
train_dataset['ds'] = x_train_state_case_df['Date']
test_dataset['ds'] = test_state_df['Date']
train_current_fatalities = train_state_df['Fatalities'].max()
train_current_cases = train_state_df['ConfirmedCases'].max()
train_min_fatalities = train_state_df['Fatalities'].min()
train_min_cases = train_state_df['ConfirmedCases'].min()
train_dataset['cap'] = train_current_cases * 1000
test_dataset['cap'] = train_current_cases * 1000
train_dataset['floor'] = 0
test_dataset['floor'] = 0
train_cases_cap = train_current_cases
if train_current_fatalities == 0 :
train_current_fatalities = 1
if train_current_cases == 0 :
train_current_cases = 1
test_dataset['cap'] = train_current_cases * 1000
if train_current_cases > 50000 :
train_dataset['cap'] = train_current_cases * 50
test_dataset['cap'] = train_current_cases * 50
if(train_current_cases > 20000 and train_current_cases <= 50000):
train_dataset['cap'] = train_current_cases * 100
test_dataset['cap'] = train_current_cases * 100
if(train_current_cases > 10000 and train_current_cases <= 20000):
train_dataset['cap'] = train_current_cases * 200
test_dataset['cap'] = train_current_cases * 200
if(train_current_cases > 5000 and train_current_cases <= 10000):
train_dataset['cap'] = train_current_cases * 400
test_dataset['cap'] = train_current_cases * 400
if(train_current_cases > 1000 and train_current_cases <= 5000):
train_dataset['cap'] = train_current_cases * 600
test_dataset['cap'] = train_current_cases * 600
if(train_current_cases > 100 and train_current_cases <= 1000):
train_dataset['cap'] = train_current_cases * 800
test_dataset['cap'] = train_current_cases * 800
if(train_current_cases > 0 and train_current_cases <= 100):
train_dataset['cap'] = train_current_cases * 1000
test_dataset['cap'] = train_current_cases * 1000
if key[0] in cluster0 :
train_dataset['cap'] = train_current_cases * cluster0_fatality_cap
test_dataset['cap'] = train_current_cases * cluster0_fatality_cap
if key[0] in cluster1 :
train_dataset['cap'] = train_current_cases * cluster1_fatality_cap
test_dataset['cap'] = train_current_cases * cluster1_fatality_cap
if key[0] in cluster2 :
train_dataset['cap'] = train_current_cases * cluster2_fatality_cap
test_dataset['cap'] = train_current_cases * cluster2_fatality_cap
if key[0] in cluster3 :
train_dataset['cap'] = train_current_cases * cluster3_fatality_cap
test_dataset['cap'] = train_current_cases * cluster3_fatality_cap
if key[0] in cluster4 :
train_dataset['cap'] = train_current_cases * cluster4_fatality_cap
test_dataset['cap'] = train_current_cases * cluster4_fatality_cap
if key[0] in cluster5 :
train_dataset['cap'] = train_current_cases * cluster5_fatality_cap
test_dataset['cap'] = train_current_cases * cluster5_fatality_cap
if key[0] in cluster7 :
train_dataset['cap'] = train_current_cases * cluster7_fatality_cap
test_dataset['cap'] = train_current_cases * cluster7_fatality_cap
if key[0] in cluster8 :
train_dataset['cap'] = train_current_cases * cluster8_fatality_cap
test_dataset['cap'] = train_current_cases * cluster8_fatality_cap
if key[0] in cluster9 :
train_dataset['cap'] = train_current_cases * cluster9_fatality_cap
test_dataset['cap'] = train_current_cases * cluster9_fatality_cap
prophet_conf.fit(train_dataset)
 y_case_predict = prophet_conf.predict(test_dataset)
 y_case_predict.rename(columns = {'trend':'ConfirmedCases'}, inplace = True)
 test_state_df = pd.merge(test_state_df, y_case_predict[['ConfirmedCases', 'ds']], left_on='Date', right_on = 'ds', how='left')
 train_dataset['y'] = train_state_df['Fatalities']
test_dataset['cap'] = train_current_fatalities * 1000
test_dataset['floor'] = 0
train_dataset['cap'] = train_current_fatalities * 1000
train_dataset['floor'] = 0
if train_current_cases > 50000 :
train_dataset['cap'] = train_current_fatalities * 50
test_dataset['cap'] = train_current_fatalities * 50
if(train_current_cases > 20000 and train_current_cases <= 50000):
train_dataset['cap'] = train_current_fatalities * 70
test_dataset['cap'] = train_current_fatalities * 70
if(train_current_cases > 10000 and train_current_cases <= 20000):
train_dataset['cap'] = train_current_fatalities * 140
test_dataset['cap'] = train_current_fatalities * 140
if(train_current_cases > 5000 and train_current_cases <= 10000):
train_dataset['cap'] = train_current_fatalities * 280
test_dataset['cap'] = train_current_fatalities * 280
if(train_current_cases > 1000 and train_current_cases <= 5000):
train_dataset['cap'] = train_current_fatalities * 420
test_dataset['cap'] = train_current_fatalities * 420
if(train_current_cases > 100 and train_current_cases <= 1000):
train_dataset['cap'] = train_current_fatalities * 560
test_dataset['cap'] = train_current_fatalities * 560
if(train_current_cases > 0 and train_current_cases <= 100):
train_dataset['cap'] = train_current_fatalities * 700
test_dataset['cap'] = train_current_fatalities * 700
if key[0] in cluster0 :
train_dataset['cap'] = train_current_fatalities * cluster0_fatality_cap
test_dataset['cap'] = train_current_fatalities * cluster0_fatality_cap
if key[0] in cluster1 :
train_dataset['cap'] = train_current_fatalities * cluster1_fatality_cap
test_dataset['cap'] = train_current_fatalities * cluster1_fatality_cap
if key[0] in cluster2 :
train_dataset['cap'] = train_current_fatalities * cluster2_fatality_cap
test_dataset['cap'] = train_current_fatalities * cluster2_fatality_cap
if key[0] in cluster3 :
train_dataset['cap'] = train_current_fatalities * cluster3_fatality_cap
test_dataset['cap'] = train_current_fatalities * cluster3_fatality_cap
if key[0] in cluster4 :
train_dataset['cap'] = train_current_fatalities * cluster4_fatality_cap
test_dataset['cap'] = train_current_fatalities * cluster4_fatality_cap
if key[0] in cluster5 :
train_dataset['cap'] = train_current_fatalities * cluster5_fatality_cap
test_dataset['cap'] = train_current_fatalities * cluster5_fatality_cap
if key[0] in cluster7 :
train_dataset['cap'] = train_current_fatalities * cluster7_fatality_cap
test_dataset['cap'] = train_current_fatalities * cluster7_fatality_cap
if key[0] in cluster8 :
train_dataset['cap'] = train_current_fatalities * cluster8_fatality_cap
test_dataset['cap'] = train_current_fatalities * cluster8_fatality_cap
if key[0] in cluster9 :
train_dataset['cap'] = train_current_fatalities * cluster9_fatality_cap
test_dataset['cap'] = train_current_fatalities * cluster9_fatality_cap
prophet_fatalities.fit(train_dataset)
 y_fatalities_predict = prophet_fatalities.predict(test_dataset)
 y_fatalities_predict.rename(columns = {'trend':'Fatalities'}, inplace = True)
 test_state_df = pd.merge(test_state_df, y_fatalities_predict[['Fatalities', 'ds']], left_on='Date', right_on = 'ds', how='left')
 predictions_dataset = predictions_dataset.append(test_state_df, ignore_index=True)
if encoding == "lb":
print(b_, "Label Encoding")
 for feat in ["cp_time", "cp_dose"]:
data_all[feat] = LabelEncoder().fit_transform(data_all[feat])
 elif encoding == "dummy":
print(b_, "One-Hot")
 data_all = pd.get_dummies(data_all, columns = ["cp_time", "cp_dose"] )<feature_engineering>
submissions = predictions_dataset.drop(['Country_Region', 'Province_State', 'Date', 'days_elapsed', 'ds_x', 'ds_y'], axis = 1)
 submissions['ConfirmedCases'] = submissions['ConfirmedCases'].astype(int)
 submissions['Fatalities'] = submissions['Fatalities'].astype(int )
GENES = [col for col in data_all.columns if col.startswith("g-")]
CELLS = [col for col in data_all.columns if col.startswith("c-")]
for stats in tqdm.tqdm(["sum", "mean", "std", "kurt", "skew"]):
data_all["g_" + stats] = getattr(data_all[GENES], stats )(axis = 1)
 data_all["c_" + stats] = getattr(data_all[CELLS], stats )(axis = 1)
 data_all["gc_" + stats] = getattr(data_all[GENES + CELLS], stats )(axis = 1 )<load_pretrained>
submissions.to_csv('submission.csv', index=False)
 print('--Done--' )