Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<load_pretrained>
!pip install -q fastprogress fastai2 fastcore fast_tabnet --upgrade
if ComputeLB:
with zipfile.ZipFile(".. /working/images.zip","r")as z:
z.extractall(".. /tmp/images2/")
 m2 = []; s2 =[]; f2 = []
user_images_unzipped_path = '.. /tmp/images2/'
images_path = [user_images_unzipped_path,'.. /input/generative-dog-images/all-dogs/all-dogs/']
public_path = '.. /input/dog-face-generation-competition-kid-metric-input/classify_image_graph_def.pb'
fid_epsilon = 10e-15
fid_value_public, distance_public, m2, s2, f2 = calculate_kid_given_paths(images_path, 'Inception', public_path, mm=m2, ss=s2, ff=f2)
 distance_public = distance_thresholding(distance_public, model_params['Inception']['cosine_distance_eps'])
 print("FID_public: ", fid_value_public, "distance_public: ", distance_public, "multiplied_public: ",
fid_value_public /(distance_public + fid_epsilon))
! rm -r.. /tmp<define_variables>
from fastai2.basics import *
from fastai2.tabular.all import *
from fast_tabnet.core import *
ver = 'logreg'<import_modules>
df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/train.csv' )
import numpy as np
import pandas as pd<set_options>
df_test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv')
 df_test.head()
warnings.simplefilter(action = 'ignore' )<import_modules>
df['key'] = df['Country_Region'] + '
df['province_flag'] = np.where(df['Province_State'].isnull() ,0,1)
 df['Province_State'] = df['Province_State'].fillna(df['Country_Region'] )
from sklearn.metrics import roc_auc_score, log_loss, accuracy_score, confusion_matrix
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.linear_model import LogisticRegression<import_modules>
df_test['key'] = df_test['Country_Region'] + '
df_test['province_flag'] = np.where(df_test['Province_State'].isnull() ,0,1)
 df_test['Province_State'] = df_test['Province_State'].fillna(df_test['Country_Region'] )
from scipy.stats import mannwhitneyu<load_from_csv>
firstconfirmed = df[(df['ConfirmedCases']>0)].groupby(['Province_State','Country_Region'])['Date'].min().reset_index()
train = pd.read_csv('.. /input/train.csv', index_col = 'id')
 train.head()<count_values>
firstfatality = df[(df['Fatalities']>0)].groupby(['Province_State','Country_Region'])['Date'].min().reset_index()
target = train['target']
train.drop('target', axis = 1, inplace = True)
 target.value_counts()<load_from_csv>
firstconfirmed.columns = ['Province_State','Country_Region','FirstCaseDate']
firstfatality.columns = ['Province_State','Country_Region','FirstFatalityDate']
test = pd.read_csv('.. /input/test.csv', index_col = 'id')
 test.head()<save_to_csv>
df = df.merge(firstconfirmed, left_on=['Province_State','Country_Region'],right_on=['Province_State','Country_Region'],how='left')
 df = df.merge(firstfatality, left_on=['Province_State','Country_Region'],right_on=['Province_State','Country_Region'],how='left' )
def save_submit(test_, clfs_, filename):
subm = pd.DataFrame(np.zeros(test_.shape[0]), index = test_.index, columns = ['target'])
 for clf in clfs_:
subm['target'] += clf.predict_proba(test_)[:, 1]
subm['target'] /= len(clfs_)
 subm = subm.reset_index()
subm.columns = ['id', 'target']
subm.to_csv(filename, index = False )<create_dataframe>
df_test = df_test.merge(firstconfirmed, left_on=['Province_State','Country_Region'],right_on=['Province_State','Country_Region'],how='left')
 df_test = df_test.merge(firstfatality, left_on=['Province_State','Country_Region'],right_on=['Province_State','Country_Region'],how='left' )
scores = pd.DataFrame(columns = ['auc', 'acc', 'loss', 'tn', 'fn', 'fp', 'tp'] )<feature_engineering>
df['Date']=pd.to_datetime(df['Date'], infer_datetime_format=True)
 df['FirstCaseDate']=pd.to_datetime(df['FirstCaseDate'], infer_datetime_format=True)
 df['FirstFatalityDate']=pd.to_datetime(df['FirstFatalityDate'], infer_datetime_format=True )
mw = pd.DataFrame(index = ['stat', 'p'])
 for c in train.columns:
mw[c] = mannwhitneyu(train[c], test[c])
 mw = mw.T
mw[mw['p'] <.01].shape<filter>
df_test['Date']=pd.to_datetime(df_test['Date'], infer_datetime_format=True)
 df_test['FirstCaseDate']=pd.to_datetime(df_test['FirstCaseDate'], infer_datetime_format=True)
 df_test['FirstFatalityDate']=pd.to_datetime(df_test['FirstFatalityDate'], infer_datetime_format=True )
bad_features = list(mw[mw['p'] <.01].index)
 len(bad_features )<split>
df['days_first_case']=(df['Date']-df['FirstCaseDate'] ).dt.days
df['days_first_fatality']=(df['Date']-df['FirstFatalityDate'] ).dt.days
def logreg_cross_validation(train_, target_, params,
num_folds = 5, repeats = 20, rs = 0):
print(params)
 clfs = []
folds = RepeatedStratifiedKFold(n_splits = num_folds, n_repeats = repeats, random_state = rs)
 valid_pred = pd.DataFrame(index = train_.index)
 for n_fold,(train_idx, valid_idx)in enumerate(folds.split(target_, target_)) :
print('--- Fold {} started at {}'.format(n_fold, time.ctime()))
train_x, train_y = train_.iloc[train_idx], target_.iloc[train_idx]
valid_x, valid_y = train_.iloc[valid_idx], target_.iloc[valid_idx]
clf = LogisticRegression(**params)
 clf.fit(train_x, train_y)
 clfs.append(clf)
 predict = clf.predict_proba(valid_x)[:, 1]
tn, fp, fn, tp = confusion_matrix(valid_y,(predict >=.5)* 1 ).ravel()
auc = roc_auc_score(valid_y, predict)
 acc = accuracy_score(valid_y,(predict >=.5)* 1)
 loss = log_loss(valid_y, predict)
 print('TN =', tn, 'FN =', fn, 'FP =', fp, 'TP =', tp)
 print('AUC = ', auc, 'Loss =', loss, 'Acc =', acc)
 valid_pred[n_fold] = pd.Series(predict, index = valid_x.index)
 del train_x, train_y, valid_x, valid_y, predict
gc.collect()
return clfs, valid_pred<init_hyperparams>
df_test['days_first_case']=(df_test['Date']-df_test['FirstCaseDate'] ).dt.days
df_test['days_first_fatality']=(df_test['Date']-df_test['FirstFatalityDate'] ).dt.days
params = {}
params['random_state'] = 0
params['n_jobs'] = -1
params['C'] =.2
params['penalty'] = 'l1'
params['class_weight'] = 'balanced'
params['solver'] = 'saga'<compute_train_metric>
df['days_first_case']=np.where(df['days_first_case']<0,0,df['days_first_case'].fillna(0))
df['days_first_fatality']=np.where(df['days_first_fatality']<0,0,df['days_first_fatality'].fillna(0))
clfs, pred = logreg_cross_validation(train.drop(bad_features, axis = 1), target, params )<prepare_output>
df_test['days_first_case']=np.where(df_test['days_first_case']<0,0,df_test['days_first_case'].fillna(0))
df_test['days_first_fatality']=np.where(df_test['days_first_fatality']<0,0,df_test['days_first_fatality'].fillna(0))
pred_mean = pred.mean(axis = 1 )<compute_test_metric>
add_datepart(df,'Date',drop=False )
scores = scores.T
tn, fp, fn, tp = confusion_matrix(target,(pred_mean >=.5)* 1 ).ravel()
scores['logreg'] = [
roc_auc_score(target, pred_mean),
accuracy_score(target,(pred_mean >=.5)* 1),
log_loss(target, pred_mean),
tn, fn, fp, tp]
scores = scores.T
scores<compute_test_metric>
add_datepart(df_test,'Date',drop=False )
score_auc = scores.loc['logreg', 'auc']
score_acc = scores.loc['logreg', 'acc']
score_loss = scores.loc['logreg', 'loss']
print(score_auc, score_acc, score_loss )<drop_column>
external = pd.read_csv('/kaggle/input/covid19-week2-external-data/external_data.csv',sep=';',decimal=',' )
loc_ver = 'v1'
filename = 'subm_{}_{}_{:.4f}_{:.4f}_{:.4f}_{}.csv'.format(ver, loc_ver, score_auc, score_acc, score_loss,
datetime.now().strftime('%Y-%m-%d'))
print(filename)
 save_submit(test.drop(bad_features, axis = 1), clfs, filename )<set_options>
df = df.merge(external, left_on='key',right_on='key',how='left' )
%matplotlib inline<load_from_csv>
df_test = df_test.merge(external, left_on='key',right_on='key',how='left' )
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv" )<drop_column>
df.pivot_table(index='Country_Region', columns='Date', values='ConfirmedCases', aggfunc=np.sum, fill_value=0 )
target = train_df["target"]
train_df = train_df.drop(["target","id"],axis=1)
 test_id = test_df["id"]
test_df = test_df.drop(["id"],axis=1 )<split>
df['Confirmedlast43'] = df['ConfirmedCases'].shift(43)
 df['Fatalitieslast43'] = df['Fatalities'].shift(43 )
train_df=reduce_mem_usage(train_df)
 test_df=reduce_mem_usage(test_df )<init_hyperparams>
df['is_valid'] = np.where(df['Date']<'2020-03-29', False, True )
shuffle_split = StratifiedShuffleSplit(test_size=0.25,train_size=0.75,n_splits=35,random_state=87951)
 param_grid = {
'alpha' : [0.1,1,10,0.01],
'l1_ratio' : np.arange(0.40,1.00,0.10),
'tol' : [0.0001,0.001]
}
eNet = ElasticNet(max_iter=10000)
 grid_search = GridSearchCV(eNet,
param_grid,
scoring='roc_auc',
cv = shuffle_split,
return_train_score=True,
n_jobs = -1)
 grid_search.fit(train_df,target )<find_best_params>
df.groupby('is_valid' ).size()
print("Best parameters : {}".format(grid_search.best_params_))
print("Best cross validation score: {:.2f}".format(grid_search.best_score_))
print("Best estimator: {}".format(grid_search.best_estimator_))<create_dataframe>
df['ConfirmedLog'] = np.log(df['ConfirmedCases']+1)
 df['FatalitiesLog'] = np.log(df['Fatalities']+1 )
results = pd.DataFrame(grid_search.cv_results_)
 results.sort_values(['mean_test_score'],ascending = False)[:10] 
<compute_test_metric>
cat_vars = ['Province_State','Country_Region','province_flag']
cont_vars = ['Elapsed',
'days_first_case',
'days_first_fatality',
'pop_density',
'population',
'area',
'lat_min',
'lat_max',
'lon_min',
'lon_max',
'centroid_x',
'centroid_y',
'wdi_country_population',
'wdi_country_arrivals',
'wdi_arrivals_per_capita',
'wdi_gini',
'wdi_perc_urban_pop',
'wdi_perc_handwashing',
'wdi_uhc_coverage',
'wdi_hospital_beds_p1000',
'wdi_smoke_prevalence',
'wdi_diabetes_prevalence',
'wdi_gdp_per_capita_ppp',
'wdi_perc_death_comm_diseases',
'wdi_perc_death_non_comm_diseases',
'wdi_death_rate_p1000',
'wdi_perc_basic_sanitation',
'wdi_dom_govmt_healt_exped_gdp',
'wdi_dom_govmt_healt_exped_per_cap',
'wdi_perc_females',
'wdi_perc_males',
'wdi_perc_female_20_29',
'wdi_perc_female_30_39',
'wdi_perc_female_40_49',
'wdi_perc_female_50_59',
'wdi_perc_female_60_69',
'wdi_perc_female_70_79',
'wdi_perc_female_80p',
'wdi_perc_male_20_29',
'wdi_perc_male_30_39',
'wdi_perc_male_40_49',
'wdi_perc_male_50_59',
'wdi_perc_male_60_69',
'wdi_perc_male_70_79',
'wdi_perc_male_80p',
'wdi_pop_denisty',
'wdi_perc_anual_growth_pop','Month','Week','Dayofyear']
dep_vars = ['ConfirmedLog']
scores_mean = np.array(results.mean_test_score ).reshape(-1)
 scores_std = np.array(results.std_test_score ).reshape(-1)
 print("mean CV scores for each fold {} ".format(scores_mean))
print("std CV scores for each fold {} ".format(scores_std))<find_best_params>
y_block = TransformBlock()
clf = grid_search.best_estimator_
type(clf )<categorify>
to = TabularPandas(df, procs, cat_names=cat_vars, cont_names=cont_vars, y_names=dep_vars, y_block=RegressionBlock() , splits=splits )
<predict_on_test>
dls = to.dataloaders(bs=512 )
pred_el = grid_search.predict(test_df )<save_to_csv>
dls.show_batch()
sub_df = pd.DataFrame()
sub_df["id"] = test_id
sub_df["target"] = pred_el
sub_df.to_csv("baseline_el.csv", index=False )<set_options>
cats, conts, y = next(iter(dls.train))
warnings.filterwarnings("ignore")
 %matplotlib inline 
<load_from_csv>
max_log_y = np.log(1.2)+ np.max(df['ConfirmedLog'])
 y_range =(0, max_log_y)
 y_range
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 print("{} observations and {} features in train set.".format(train.shape[0],train.shape[1]))
print("{} observations and {} features in test set.".format(test.shape[0],test.shape[1]))<concatenate>
learn = tabular_learner(dls, layers=[300,250], loss_func=MSELossFlat() ,
config=tabular_config(ps=[0.005,0.015]), y_range=y_range,
metrics=[exp_rmspe, rmse] )
train_test = pd.concat([train[train.columns[2:]], test[test.columns[1:]]] )<compute_test_metric>
learn.model
AUCs = []
Ginis = []
for i in range(300):
AUC = roc_auc_score(train.target.values, train[str(i)].values)
 AUCs.append(AUC)
 Gini = 2*AUC - 1
Ginis.append(Gini)
<sort_values>
learn.load('/kaggle/input/covid19-week2-external-data/best18_2day' )
np.sort(np.abs(Ginis)) [::-1]<sort_values>
preds, y = learn.get_preds()
np.argsort(np.abs(Ginis)) [::-1]<sort_values>
np.exp(preds)-1
[Ginis[k] for k in np.argsort(np.abs(Ginis)) [::-1][:13]]<compute_test_metric>
np.exp(y)-1
roc_auc_score(train.target.values,train['33'].values )<compute_test_metric>
dl = learn.dls.test_dl(df )
roc_auc_score(train.target.values,train['65'].values )<compute_test_metric>
raw_test_preds = learn.get_preds(dl=dl )
roc_auc_score(train.target.values,-train['217'].values )<compute_test_metric>
preds = np.exp(to_np(raw_test_preds[0])) -1
roc_auc_score(train.target.values,-train['117'].values )<compute_test_metric>
preds = pd.DataFrame(preds)
 preds.columns = ['pred_confirmed']
roc_auc_score(train.target.values,-train['91'].values )<compute_test_metric>
df = pd.concat([df, preds], axis=1 )
roc_auc_score(train.target.values,-train['295'].values )<compute_test_metric>
dl = learn.dls.test_dl(df_test )
roc_auc_score(train.target.values,train['24'].values )<compute_test_metric>
raw_test_preds = learn.get_preds(dl=dl )
roc_auc_score(train.target.values,train['199'].values )<compute_test_metric>
preds_confirmed_test = np.exp(to_np(raw_test_preds[0])) -1
roc_auc_score(train.target.values,-train['80'].values )<compute_test_metric>
cat_vars = ['Province_State','Country_Region','province_flag']
cont_vars = ['Elapsed',
'days_first_case',
'days_first_fatality',
'pop_density',
'population',
'area',
'lat_min',
'lat_max',
'lon_min',
'lon_max',
'centroid_x',
'centroid_y',
'wdi_country_population',
'wdi_country_arrivals',
'wdi_arrivals_per_capita',
'wdi_gini',
'wdi_perc_urban_pop',
'wdi_perc_handwashing',
'wdi_uhc_coverage',
'wdi_hospital_beds_p1000',
'wdi_smoke_prevalence',
'wdi_diabetes_prevalence',
'wdi_gdp_per_capita_ppp',
'wdi_perc_death_comm_diseases',
'wdi_perc_death_non_comm_diseases',
'wdi_death_rate_p1000',
'wdi_perc_basic_sanitation',
'wdi_dom_govmt_healt_exped_gdp',
'wdi_dom_govmt_healt_exped_per_cap',
'wdi_perc_females',
'wdi_perc_males',
'wdi_perc_female_20_29',
'wdi_perc_female_30_39',
'wdi_perc_female_40_49',
'wdi_perc_female_50_59',
'wdi_perc_female_60_69',
'wdi_perc_female_70_79',
'wdi_perc_female_80p',
'wdi_perc_male_20_29',
'wdi_perc_male_30_39',
'wdi_perc_male_40_49',
'wdi_perc_male_50_59',
'wdi_perc_male_60_69',
'wdi_perc_male_70_79',
'wdi_perc_male_80p',
'wdi_pop_denisty',
'wdi_perc_anual_growth_pop','Month','Week','Dayofyear','pred_confirmed']
dep_vars = ['FatalitiesLog']
roc_auc_score(train.target.values,-train['73'].values )<compute_test_metric>
to = TabularPandas(df, procs, cat_names=cat_vars, cont_names=cont_vars, y_names=dep_vars, y_block=RegressionBlock() , splits=splits )
roc_auc_score(train.target.values,-train['194'].values )<compute_test_metric>
dls = to.dataloaders(bs=512 )
roc_auc_score(train.target.values,0.146*train['33'].values + 0.12*train['65'].values-0.06*train['217'].values-0.05*train['117'].values
-0.05*train['91'].values-0.05*train['295'].values+0.05*train['24'].values+0.05*train['199'].values-
0.05*train['80'].values- 0.05*train['73'].values-0.05*train['194'].values )<define_variables>
max_log_y = np.log(1.2)+ np.max(df['FatalitiesLog'])
 y_range =(0, max_log_y)
 y_range
preds =(0.146*test['33'].values + 0.12*test['65'].values-0.06*test['217'].values-0.05*test['117'].values
-0.05*test['91'].values-0.05*test['295'].values+0.05*test['24'].values+0.05*test['199'].values-
0.05*test['80'].values- 0.05*test['73'].values-0.05*test['194'].values)
 preds = rankdata(preds)/preds.shape[0]
preds<save_to_csv>
learn = tabular_learner(dls, layers=[300,250], loss_func=MSELossFlat() ,
config=tabular_config(ps=[0.005,0.015]), y_range=y_range,
metrics=[exp_rmspe, rmse] )
sample_submission = pd.read_csv('.. /input/sample_submission.csv')
 sample_submission['target'] = preds
sample_submission.to_csv('submission.csv', index=False )<train_model>
learn.load('/kaggle/input/covid19-week2-external-data/fat_best2_2day' )
pca = PCA(n_components=0.99)
 pca.fit(train_test.values )<find_best_score>
preds, y = learn.get_preds()
pca.n_components_<train_model>
preds_confirmed_test = pd.DataFrame(preds_confirmed_test )
pca = PCA(n_components=0.9)
 pca.fit(train_test.values)
 pca.n_components_<train_on_grid>
preds_confirmed_test.columns=['ConfirmedCases']
Sum_of_squared_distances = []
K = range(1,15)
 for k in K:
km = KMeans(n_clusters=k)
 km = km.fit(train_test)
 Sum_of_squared_distances.append(km.inertia_ )<compute_train_metric>
df_test = pd.concat([df_test, preds_confirmed_test], axis=1 )
NN = 80
train_pred = 0
gini_list = [Ginis[k] for k in np.argsort(np.abs(Ginis)) [::-1][:NN]]
for i in range(NN):
if gini_list[i] > 0:
train_pred += train[str(np.argsort(np.abs(Ginis)) [::-1][i])].values
else:
train_pred -= train[str(np.argsort(np.abs(Ginis)) [::-1][i])].values
roc_auc_score(train.target.values, train_pred )<sort_values>
df_test['pred_confirmed']=df_test['ConfirmedCases']
test_pred = 0
gini_list = [Ginis[k] for k in np.argsort(np.abs(Ginis)) [::-1][:NN]]
for i in range(NN):
if gini_list[i] > 0:
test_pred += test[str(np.argsort(np.abs(Ginis)) [::-1][i])].values
else:
test_pred -= test[str(np.argsort(np.abs(Ginis)) [::-1][i])].values<compute_test_metric>
dl = learn.dls.test_dl(df_test )
test_pred = rankdata(test_pred)/test_pred.shape[0]
test_pred<save_to_csv>
raw_test_preds = learn.get_preds(dl=dl )
sample_submission['target'] = test_pred
sample_submission.to_csv('submission_80.csv', index=False )<set_options>
preds_fatalities_test = np.exp(to_np(raw_test_preds[0])) -1
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore' )<drop_column>
preds_fatalities_test = pd.DataFrame(preds_fatalities_test )
target_col = ["target"]
id_dataset = ["id"]
num_cols = [x for x in data.columns if x not in target_col + id_dataset]
std = StandardScaler()
scaled = std.fit_transform(data[num_cols])
 scaled = pd.DataFrame(scaled,columns=num_cols)
 df_data_og = data.copy()
data = data.drop(columns = num_cols,axis = 1)
 data = data.merge(scaled,left_index=True,right_index=True,how = "left")
 data = data.drop(columns = ['id'],axis = 1 )<drop_column>
preds_fatalities_test.columns=['Fatalities']
test = pd.read_csv('.. /input/test.csv')
 target_col = ["target"]
id_dataset = ["id"]
num_cols = [x for x in test.columns if x not in target_col + id_dataset]
std = StandardScaler()
scaled = std.fit_transform(test[num_cols])
 scaled = pd.DataFrame(scaled,columns=num_cols)
 df_test_og = test.copy()
test = test.drop(columns = num_cols,axis = 1)
 test = test.merge(scaled,left_index=True,right_index=True,how = "left")
 test = test.drop(columns = ['id'],axis = 1 )<prepare_x_and_y>
df_test = pd.concat([df_test, preds_fatalities_test], axis=1 )
X = data.drop('target', 1)
 y = data['target']<train_on_grid>
sub_ex = df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/submission.csv' )
random_state = 42
log_clf = LogisticRegression(random_state = random_state)
 param_grid = {'class_weight' : ['balanced', None],
'penalty' : ['l2','l1'],
'C' : [0.001, 0.01, 0.1, 1, 10, 100, 1000]}
grid = GridSearchCV(estimator = log_clf, param_grid = param_grid , scoring = 'roc_auc', verbose = 1, n_jobs = -1)
 grid.fit(X,y)
 print("Best Score:" + str(grid.best_score_))
print("Best Parameters: " + str(grid.best_params_))
best_parameters = grid.best_params_<train_model>
sub = df_test[['ForecastId','ConfirmedCases','Fatalities']]
log_clf = LogisticRegression(**best_parameters)
 log_clf.fit(X,y)
 selector = RFE(log_clf, 25, step=1)
 selector.fit(X,y)
 scores_table(selector, 'selector_clf' )<save_to_csv>
sub.to_csv('submission.csv',index=False )