Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<split>
pd.set_option('max_columns', 500)
 pd.set_option('max_rows', 500)
 pd.options.display.float_format = '{:.2f}'.format
X_training, X_valid, y_training, y_valid = train_test_split(X_train, y_train, test_size=0.2, random_state=0 )<import_modules>
train = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 tt = pd.concat([train, test], sort=False)
 tt = train.merge(test, on=['Province_State','Country_Region','Date'], how='outer')
 def name_place(x):
try:
x_new = x['Country_Region'] + "_" + x['Province_State']
except:
x_new = x['Country_Region']
return x_new
tt['Place'] = tt.apply(lambda x: name_place(x), axis=1)
 tt['Date'] = pd.to_datetime(tt['Date'])
 tt['doy'] = tt['Date'].dt.dayofyear
tt['dow'] = tt['Date'].dt.dayofweek
tt['hasProvidence'] = ~tt['Province_State'].isna()
country_meta = pd.read_csv('.. /input/covid19-forecasting-metadata/region_metadata.csv')
 tt = tt.merge(country_meta, how='left')
 country_date_meta = pd.read_csv('.. /input/covid19-forecasting-metadata/region_date_metadata.csv')
 tt['HasFatality'] = tt.groupby('Place')['Fatalities'].transform(lambda x: x.max() > 0)
 tt['HasCases'] = tt.groupby('Place')['ConfirmedCases'].transform(lambda x: x.max() > 0)
 first_case_date = tt.query('ConfirmedCases >= 1' ).groupby('Place')['Date'].min().to_dict()
ten_case_date = tt.query('ConfirmedCases >= 10' ).groupby('Place')['Date'].min().to_dict()
hundred_case_date = tt.query('ConfirmedCases >= 100' ).groupby('Place')['Date'].min().to_dict()
first_fatal_date = tt.query('Fatalities >= 1' ).groupby('Place')['Date'].min().to_dict()
ten_fatal_date = tt.query('Fatalities >= 10' ).groupby('Place')['Date'].min().to_dict()
hundred_fatal_date = tt.query('Fatalities >= 100' ).groupby('Place')['Date'].min().to_dict()
tt['First_Case_Date'] = tt['Place'].map(first_case_date)
 tt['Ten_Case_Date'] = tt['Place'].map(ten_case_date)
 tt['Hundred_Case_Date'] = tt['Place'].map(hundred_case_date)
 tt['First_Fatal_Date'] = tt['Place'].map(first_fatal_date)
 tt['Ten_Fatal_Date'] = tt['Place'].map(ten_fatal_date)
 tt['Hundred_Fatal_Date'] = tt['Place'].map(hundred_fatal_date)
 tt['Days_Since_First_Case'] =(tt['Date'] - tt['First_Case_Date'] ).dt.days
tt['Days_Since_Ten_Cases'] =(tt['Date'] - tt['Ten_Case_Date'] ).dt.days
tt['Days_Since_Hundred_Cases'] =(tt['Date'] - tt['Hundred_Case_Date'] ).dt.days
tt['Days_Since_First_Fatal'] =(tt['Date'] - tt['First_Fatal_Date'] ).dt.days
tt['Days_Since_Ten_Fatal'] =(tt['Date'] - tt['Ten_Fatal_Date'] ).dt.days
tt['Days_Since_Hundred_Fatal'] =(tt['Date'] - tt['Hundred_Fatal_Date'] ).dt.days
smoking = pd.read_csv(".. /input/smokingstats/share-of-adults-who-smoke.csv")
 smoking = smoking.rename(columns={'Smoking prevalence, total(ages 15+ )(% of adults)': 'Smoking_Rate'})
 smoking_dict = smoking.groupby('Entity')['Year'].max().to_dict()
smoking['LastYear'] = smoking['Entity'].map(smoking_dict)
 smoking = smoking.query('Year == LastYear' ).reset_index()
smoking['Entity'] = smoking['Entity'].str.replace('United States', 'US')
 tt = tt.merge(smoking[['Entity','Smoking_Rate']],
left_on='Country_Region',
right_on='Entity',
how='left',
validate='m:1')\ 
.drop('Entity', axis=1)
 country_info = pd.read_csv('.. /input/countryinfo/covid19countryinfo.csv')
 tt = tt.merge(country_info, left_on=['Country_Region','Province_State'],
right_on=['country','region'],
how='left',
validate='m:1')
 us_state_info = pd.read_html('https://simple.wikipedia.org/wiki/List_of_U.S._states_by_population')[0] \
[['State','Population estimate, July 1, 2019[2]']] \ 
.rename(columns={'Population estimate, July 1, 2019[2]' : 'Population'})
 tt = tt.merge(us_state_info[['State','Population']],
left_on='Province_State',
right_on='State',
how='left')
 tt['pop'] = pd.to_numeric(tt['pop'].str.replace(',',''))
tt['pop'] = tt['pop'].fillna(tt['Population'])
 tt['pop'] = pd.to_numeric(tt['pop'])
 tt['pop_diff'] = tt['pop'] - tt['Population']
tt['Population_final'] = tt['Population']
tt.loc[~tt['hasProvidence'], 'Population_final'] = tt.loc[~tt['hasProvidence']]['pop']
tt['Confirmed_Cases_Diff'] = tt.groupby('Place')['ConfirmedCases'].diff()
tt['Fatailities_Diff'] = tt.groupby('Place')['Fatalities'].diff()
max_date = tt.dropna(subset=['ConfirmedCases'])['Date'].max()
tt['gdp2019'] = pd.to_numeric(tt['gdp2019'].str.replace(',',''))
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.svm import SVC, LinearSVC
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.metrics import accuracy_score
from sklearn.model_selection import GridSearchCV<find_best_params>
pop_dict = {'Angola': int(29.78 * 10**6),
'Australia_Australian Capital Territory': 423_800,
'Australia_New South Wales': int(7.544 * 10**6),
'Australia_Northern Territory': 244_300,
'Australia_Queensland' : int(5.071 * 10**6),
'Australia_South Australia' : int(1.677 * 10**6),
'Australia_Tasmania': 515_000,
'Australia_Victoria': int(6.359 * 10**6),
'Australia_Western Australia': int(2.589 * 10**6),
'Brazil': int(209.3 * 10**6),
'Canada_Alberta' : int(4.371 * 10**6),
'Canada_British Columbia' : int(5.071 * 10**6),
'Canada_Manitoba' : int(1.369 * 10**6),
'Canada_New Brunswick' : 776_827,
'Canada_Newfoundland and Labrador' : 521_542,
'Canada_Nova Scotia' : 971_395,
'Canada_Ontario' : int(14.57 * 10**6),
'Canada_Prince Edward Island' : 156_947,
'Canada_Quebec' : int(8.485 * 10**6),
'Canada_Saskatchewan': int(1.174 * 10**6),
'China_Anhui': int(62 * 10**6),
'China_Beijing': int(21.54 * 10**6),
'China_Chongqing': int(30.48 * 10**6),
'China_Fujian' : int(38.56 * 10**6),
'China_Gansu' : int(25.58 * 10**6),
'China_Guangdong' : int(113.46 * 10**6),
'China_Guangxi' : int(48.38 * 10**6),
'China_Guizhou' : int(34.75 * 10**6),
'China_Hainan' : int(9.258 * 10**6),
'China_Hebei' : int(74.7 * 10**6),
'China_Heilongjiang' : int(38.31 * 10**6),
'China_Henan' : int(94 * 10**6),
'China_Hong Kong' : int(7.392 * 10**6),
'China_Hubei' : int(58.5 * 10**6),
'China_Hunan' : int(67.37 * 10**6),
'China_Inner Mongolia' : int(24.71 * 10**6),
'China_Jiangsu' : int(80.4 * 10**6),
'China_Jiangxi' : int(45.2 * 10**6),
'China_Jilin' : int(27.3 * 10**6),
'China_Liaoning' : int(43.9 * 10**6),
'China_Macau' : 622_567,
'China_Ningxia' : int(6.301 * 10**6),
'China_Qinghai' : int(5.627 * 10**6),
'China_Shaanxi' : int(37.33 * 10**6),
'China_Shandong' : int(92.48 * 10**6),
'China_Shanghai' : int(24.28 * 10**6),
'China_Shanxi' : int(36.5 * 10**6),
'China_Sichuan' : int(81.1 * 10**6),
'China_Tianjin' : int(15 * 10**6),
'China_Tibet' : int(3.18 * 10**6),
'China_Xinjiang' : int(21.81 * 10**6),
'China_Yunnan' : int(45.97 * 10**6),
'China_Zhejiang' : int(57.37 * 10**6),
'Denmark_Faroe Islands' : 51_783,
'Denmark_Greenland' : 56_171,
'France_French Guiana' : 290_691,
'France_French Polynesia' : 283_007,
'France_Guadeloupe' : 395_700,
'France_Martinique' : 376_480,
'France_Mayotte' : 270_372,
'France_New Caledonia' : 99_926,
'France_Reunion' : 859_959,
'France_Saint Barthelemy' : 9_131,
'France_St Martin' : 32_125,
'Netherlands_Aruba' : 105_264,
'Netherlands_Curacao' : 161_014,
'Netherlands_Sint Maarten' : 41_109,
'Papua New Guinea' : int(8.251 * 10**6),
'US_Guam' : 164_229,
'US_Virgin Islands' : 107_268,
'United Kingdom_Bermuda' : 65_441,
'United Kingdom_Cayman Islands' : 61_559,
'United Kingdom_Channel Islands' : 170_499,
'United Kingdom_Gibraltar' : 34_571,
'United Kingdom_Isle of Man' : 84_287,
'United Kingdom_Montserrat' : 4_922
}
tt['Population_final'] = tt['Population_final'].fillna(tt['Place'].map(pop_dict))
params = {"C": [0.5, 1.0, 1.5],
"gamma": [0.01, 0.05, 0.1],
"probability": [True]}
svc_accuracy, svc_clf = modeling(params, SVC() )<choose_model_class>
tt.loc[tt['Place'] == 'Diamond Princess', 'Population final'] = 2_670
params = {"C": [0.1, 1, 10],
"max_iter": [50, 100, 200]}
lr_accuracy, lr_clf = modeling(params, LogisticRegression() )<choose_model_class>
tt['ConfirmedCases_Log'] = tt['ConfirmedCases'].apply(np.log1p)
 tt['Fatalities_Log'] = tt['Fatalities'].apply(np.log1p )
params = {"n_neighbors": [2, 3, 4, 5, 10, 15],
"leaf_size": [20, 30, 50],
"weights": ["uniform", "distance"],
"algorithm": ["auto", "ball_tree", "kd_tree"]}
knc_accuracy, knc_clf = modeling(params, KNeighborsClassifier() )<find_best_params>
tt['Population_final'] = tt['Population_final'].astype('int')
 tt['Cases_Per_100kPop'] =(tt['ConfirmedCases'] / tt['Population_final'])* 100000
tt['Fatalities_Per_100kPop'] =(tt['Fatalities'] / tt['Population_final'])* 100000
tt['Cases_Percent_Pop'] =(( tt['ConfirmedCases'] / tt['Population_final'])* 100)
 tt['Fatalities_Percent_Pop'] =(( tt['Fatalities'] / tt['Population_final'])* 100)
 tt['Cases_Log_Percent_Pop'] =(( tt['ConfirmedCases'] / tt['Population_final'])* 100 ).apply(np.log1p)
 tt['Fatalities_Log_Percent_Pop'] =(( tt['Fatalities'] / tt['Population_final'])* 100 ).apply(np.log1p)
 tt['Max_Confirmed_Cases'] = tt.groupby('Place')['ConfirmedCases'].transform(max)
 tt['Max_Fatalities'] = tt.groupby('Place')['Fatalities'].transform(max)
 tt['Max_Cases_Per_100kPop'] = tt.groupby('Place')['Cases_Per_100kPop'].transform(max)
 tt['Max_Fatalities_Per_100kPop'] = tt.groupby('Place')['Fatalities_Per_100kPop'].transform(max )
params = {}
gnb_accuracy, gnb_clf = modeling(params, GaussianNB() )<find_best_params>
tt.query('Date == @max_date')[['Place','Max_Cases_Per_100kPop',
'Max_Fatalities_Per_100kPop','Max_Confirmed_Cases',
'Population_final',
'Days_Since_First_Case',
'Confirmed_Cases_Diff']] \ 
.drop_duplicates() \ 
.sort_values('Max_Cases_Per_100kPop', ascending=False )
params = {"C": [0.005, 0.01, 0.5, 1.0]}
lsvc_accuracy, lsvc_clf = modeling(params, LinearSVC() )<choose_model_class>
tt['Past_7Days_ConfirmedCases_Std'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200324' ).groupby('Place')['ConfirmedCases'].std().to_dict())
 tt['Past_7Days_Fatalities_Std'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200324' ).groupby('Place')['Fatalities'].std().to_dict())
 tt['Past_7Days_ConfirmedCases_Min'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200324' ).groupby('Place')['ConfirmedCases'].min().to_dict())
 tt['Past_7Days_Fatalities_Min'] = tt['Place'].map(tt.dropna(subset=['Fatalities'] ).query('Date >= 20200324' ).groupby('Place')['Fatalities'].min().to_dict())
 tt['Past_7Days_ConfirmedCases_Max'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200324' ).groupby('Place')['ConfirmedCases'].max().to_dict())
 tt['Past_7Days_Fatalities_Max'] = tt['Place'].map(tt.dropna(subset=['Fatalities'] ).query('Date >= 20200324' ).groupby('Place')['Fatalities'].max().to_dict())
 tt['Past_7Days_Confirmed_Change_of_Total'] =(tt['Past_7Days_ConfirmedCases_Max'] - tt['Past_7Days_ConfirmedCases_Min'])/(tt['Past_7Days_ConfirmedCases_Max'])
 tt['Past_7Days_Fatalities_Change_of_Total'] =(tt['Past_7Days_Fatalities_Max'] - tt['Past_7Days_Fatalities_Min'])/(tt['Past_7Days_Fatalities_Max'] )
params = {"learning_rate": [0.01, 0.03, 0.05, 0.1],
"n_estimators": [30, 50, 100]}
gbc_accuracy, gbc_clf = modeling(params, GradientBoostingClassifier() )<create_dataframe>
tt['Past_21Days_ConfirmedCases_Std'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200310' ).groupby('Place')['ConfirmedCases'].std().to_dict())
 tt['Past_21Days_Fatalities_Std'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200310' ).groupby('Place')['Fatalities'].std().to_dict())
 tt['Past_21Days_ConfirmedCases_Min'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200310' ).groupby('Place')['ConfirmedCases'].min().to_dict())
 tt['Past_21Days_Fatalities_Min'] = tt['Place'].map(tt.dropna(subset=['Fatalities'] ).query('Date >= 20200324' ).groupby('Place')['Fatalities'].min().to_dict())
 tt['Past_21Days_ConfirmedCases_Max'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200310' ).groupby('Place')['ConfirmedCases'].max().to_dict())
 tt['Past_21Days_Fatalities_Max'] = tt['Place'].map(tt.dropna(subset=['Fatalities'] ).query('Date >= 20200310' ).groupby('Place')['Fatalities'].max().to_dict())
 tt['Past_21Days_Confirmed_Change_of_Total'] =(tt['Past_21Days_ConfirmedCases_Max'] - tt['Past_21Days_ConfirmedCases_Min'])/(tt['Past_21Days_ConfirmedCases_Max'])
 tt['Past_21Days_Fatalities_Change_of_Total'] =(tt['Past_21Days_Fatalities_Max'] - tt['Past_21Days_Fatalities_Min'])/(tt['Past_21Days_Fatalities_Max'])
 tt['Past_7Days_Fatalities_Change_of_Total'] = tt['Past_7Days_Fatalities_Change_of_Total'].fillna(0)
 tt['Past_21Days_Fatalities_Change_of_Total'] = tt['Past_21Days_Fatalities_Change_of_Total'].fillna(0 )
accuracy = pd.DataFrame({"model": ["RandomForestClassifier", "SVC", "LogisticRegression", "KNeighborsClassifier", "GaussianNB", "LinearSVC", "GradientBoostingClassifier"],
"accuracy": [rfc_accuracy, svc_accuracy, lr_accuracy, knc_accuracy, gnb_accuracy, lsvc_accuracy, gbc_accuracy]})
 accuracy.sort_values(by="accuracy", ascending=False )<train_model>
tt['Date_7Days_Since_First_Case'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Case'] == 7] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_14Days_Since_First_Case'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Case'] == 14] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_21Days_Since_First_Case'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Case'] == 21] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_28Days_Since_First_Case'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Case'] == 28] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_35Days_Since_First_Case'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Case'] == 35] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_60Days_Since_First_Case'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Case'] == 60] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_7Days_Since_Ten_Cases'] = tt['Place'].map(tt.loc[tt['Days_Since_Ten_Cases'] == 7] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_14Days_Since_Ten_Cases'] = tt['Place'].map(tt.loc[tt['Days_Since_Ten_Cases'] == 14] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_21Days_Since_Ten_Cases'] = tt['Place'].map(tt.loc[tt['Days_Since_Ten_Cases'] == 21] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_28Days_Since_Ten_Cases'] = tt['Place'].map(tt.loc[tt['Days_Since_Ten_Cases'] == 28] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_35Days_Since_Ten_Cases'] = tt['Place'].map(tt.loc[tt['Days_Since_Ten_Cases'] == 35] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_60Days_Since_Ten_Cases'] = tt['Place'].map(tt.loc[tt['Days_Since_Ten_Cases'] == 60] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_7Days_Since_First_Fatal'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Fatal'] == 7] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_14Days_Since_First_Fatal'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Fatal'] == 14] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_21Days_Since_First_Fatal'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Fatal'] == 21] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_28Days_Since_First_Fatal'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Fatal'] == 28] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_35Days_Since_First_Fatal'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Fatal'] == 35] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_60Days_Since_First_Fatal'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Fatal'] == 60] \ 
.set_index('Place')['Date'] \ 
.to_dict() )
vt_clf = VotingClassifier(estimators=[("lr", lr_clf),("svc", svc_clf),("gbc", gbc_clf)], voting="soft")
 vt_clf.fit(X_training, y_training)
 print("accuracy:", accuracy_score(vt_clf.predict(X_valid), y_valid))<save_to_csv>
tt['CC_7D_1stCase'] = tt.loc[tt['Date_7Days_Since_First_Case'] == tt['Date']]['ConfirmedCases']
tt['CC_14D_1stCase'] = tt.loc[tt['Date_14Days_Since_First_Case'] == tt['Date']]['ConfirmedCases']
tt['CC_21D_1stCase'] = tt.loc[tt['Date_21Days_Since_First_Case'] == tt['Date']]['ConfirmedCases']
tt['CC_28D_1stCase'] = tt.loc[tt['Date_28Days_Since_First_Case'] == tt['Date']]['ConfirmedCases']
tt['CC_35D_1stCase'] = tt.loc[tt['Date_35Days_Since_First_Case'] == tt['Date']]['ConfirmedCases']
tt['CC_60D_1stCase'] = tt.loc[tt['Date_60Days_Since_First_Case'] == tt['Date']]['ConfirmedCases']
tt['F_7D_1stCase'] = tt.loc[tt['Date_7Days_Since_First_Case'] == tt['Date']]['Fatalities']
tt['F_14D_1stCase'] = tt.loc[tt['Date_14Days_Since_First_Case'] == tt['Date']]['Fatalities']
tt['F_21D_1stCase'] = tt.loc[tt['Date_21Days_Since_First_Case'] == tt['Date']]['Fatalities']
tt['F_28D_1stCase'] = tt.loc[tt['Date_28Days_Since_First_Case'] == tt['Date']]['Fatalities']
tt['F_35D_1stCase'] = tt.loc[tt['Date_35Days_Since_First_Case'] == tt['Date']]['Fatalities']
tt['F_60D_1stCase'] = tt.loc[tt['Date_60Days_Since_First_Case'] == tt['Date']]['Fatalities']
tt['CC_7D_10Case'] = tt.loc[tt['Date_7Days_Since_Ten_Cases'] == tt['Date']]['ConfirmedCases']
tt['CC_14D_10Case'] = tt.loc[tt['Date_14Days_Since_Ten_Cases'] == tt['Date']]['ConfirmedCases']
tt['CC_21D_10Case'] = tt.loc[tt['Date_21Days_Since_Ten_Cases'] == tt['Date']]['ConfirmedCases']
tt['CC_28D_10Case'] = tt.loc[tt['Date_28Days_Since_Ten_Cases'] == tt['Date']]['ConfirmedCases']
tt['CC_35D_10Case'] = tt.loc[tt['Date_35Days_Since_Ten_Cases'] == tt['Date']]['ConfirmedCases']
tt['CC_60D_10Case'] = tt.loc[tt['Date_60Days_Since_Ten_Cases'] == tt['Date']]['ConfirmedCases']
tt['F_7D_10Case'] = tt.loc[tt['Date_7Days_Since_Ten_Cases'] == tt['Date']]['Fatalities']
tt['F_14D_10Case'] = tt.loc[tt['Date_14Days_Since_Ten_Cases'] == tt['Date']]['Fatalities']
tt['F_21D_10Case'] = tt.loc[tt['Date_21Days_Since_Ten_Cases'] == tt['Date']]['Fatalities']
tt['F_28D_10Case'] = tt.loc[tt['Date_28Days_Since_Ten_Cases'] == tt['Date']]['Fatalities']
tt['F_35D_10Case'] = tt.loc[tt['Date_35Days_Since_Ten_Cases'] == tt['Date']]['Fatalities']
tt['F_60D_10Case'] = tt.loc[tt['Date_60Days_Since_Ten_Cases'] == tt['Date']]['Fatalities']
tt['CC_7D_1Fatal'] = tt.loc[tt['Date_7Days_Since_First_Fatal'] == tt['Date']]['ConfirmedCases']
tt['CC_14D_1Fatal'] = tt.loc[tt['Date_14Days_Since_First_Fatal'] == tt['Date']]['ConfirmedCases']
tt['CC_21D_1Fatal'] = tt.loc[tt['Date_21Days_Since_First_Fatal'] == tt['Date']]['ConfirmedCases']
tt['CC_28D_1Fatal'] = tt.loc[tt['Date_28Days_Since_First_Fatal'] == tt['Date']]['ConfirmedCases']
tt['CC_35D_1Fatal'] = tt.loc[tt['Date_35Days_Since_First_Fatal'] == tt['Date']]['ConfirmedCases']
tt['CC_60D_1Fatal'] = tt.loc[tt['Date_60Days_Since_First_Fatal'] == tt['Date']]['ConfirmedCases']
tt['F_7D_1Fatal'] = tt.loc[tt['Date_7Days_Since_First_Fatal'] == tt['Date']]['Fatalities']
tt['F_14D_1Fatal'] = tt.loc[tt['Date_14Days_Since_First_Fatal'] == tt['Date']]['Fatalities']
tt['F_21D_1Fatal'] = tt.loc[tt['Date_21Days_Since_First_Fatal'] == tt['Date']]['Fatalities']
tt['F_28D_1Fatal'] = tt.loc[tt['Date_28Days_Since_First_Fatal'] == tt['Date']]['Fatalities']
tt['F_35D_1Fatal'] = tt.loc[tt['Date_35Days_Since_First_Fatal'] == tt['Date']]['Fatalities']
tt['F_60D_1Fatal'] = tt.loc[tt['Date_60Days_Since_First_Fatal'] == tt['Date']]['Fatalities']
lr_clf.fit(X_train, y_train)
 submission_prediction = lr_clf.predict(X_test)
 submission = pd.DataFrame({"id": Id, "type": submission_prediction})
 submission.to_csv("submission.csv", index=False )<define_variables>
latest_summary_stats = tt.query('Date == @max_date')\
[['Country_Region',
'Place',
'Max_Cases_Per_100kPop',
'Max_Fatalities_Per_100kPop',
'Max_Confirmed_Cases',
'Population_final',
'Days_Since_First_Case',
'Days_Since_Ten_Cases']] \ 
.drop_duplicates()
MAX_ROUNDS = 650
OPTIMIZE_ROUNDS = False
LEARNING_RATE = 0.05<import_modules>
from sklearn.linear_model import LinearRegression, ElasticNet
import numpy as np
import pandas as pd
from catboost import CatBoostClassifier
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from numba import jit<compute_train_metric>
for myplace in us_states:
try:
fig, axs = plt.subplots(1, 2, figsize=(15, 3))
dat = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','ConfirmedCases_Log']].dropna()
X = dat['Days_Since_Ten_Cases']
y = dat['ConfirmedCases_Log']
y = y.cummax()
dat_all = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','ConfirmedCases_Log']]
X_pred = dat_all['Days_Since_Ten_Cases']
en = ElasticNet()
en.fit(X.values.reshape(-1, 1), y.values)
 preds = en.predict(X_pred.values.reshape(-1, 1))
tt.loc[(tt['Place'] == myplace)&(tt['Days_Since_Ten_Cases'] >= 0), 'ConfirmedCases_Log_Pred1'] = preds
tt.loc[(tt['Place'] == myplace), 'ConfirmedCases_Pred1'] = tt['ConfirmedCases_Log_Pred1'].apply(np.expm1)
 pop_myplace = tt.query('Place == @myplace')['Population_final'].values[0]
tt.loc[(tt['Place'] == myplace)&(tt['ConfirmedCases_Pred1'] >(0.05 * pop_myplace)) , 'ConfirmedCases_Pred1'] =(0.05 * pop_myplace)
 ax = tt.query('Place == @myplace' ).set_index('Date')[['ConfirmedCases','ConfirmedCases_Pred1']].plot(figsize=(15, 5), title=myplace, ax=axs[0])
 dat = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','Fatalities_Log']].dropna()
if len(dat)< 5:
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt.loc[(tt['Place'] == myplace)]['ConfirmedCases_Pred1'] * 0.0001
elif tt.query('Place == @myplace')['Fatalities'].max() < 5:
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt.loc[(tt['Place'] == myplace)]['ConfirmedCases_Pred1'] * 0.0001
else:
X = dat['Days_Since_Ten_Cases']
y = dat['Fatalities_Log']
y = y.cummax()
dat_all = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','Fatalities_Log']]
X_pred = dat_all['Days_Since_Ten_Cases']
en = ElasticNet()
en.fit(X.values.reshape(-1, 1), y.values)
 preds = en.predict(X_pred.values.reshape(-1, 1))
tt.loc[(tt['Place'] == myplace)&(tt['Days_Since_Ten_Cases'] >= 0), 'Fatalities_Log_Pred1'] = preds
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt['Fatalities_Log_Pred1'].apply(np.expm1)
 pop_myplace = tt.query('Place == @myplace')['Population_final'].values[0]
tt.loc[(tt['Place'] == myplace)&(tt['Fatalities_Pred1'] >(0.0001 * pop_myplace)) , 'Fatalities_Pred1'] =(0.0001 * pop_myplace)
 ax = tt.query('Place == @myplace' ).set_index('Date')[['Fatalities','Fatalities_Pred1']].plot(figsize=(15, 5), title=myplace, ax=axs[1])
 plt.show()
except:
print(f'============= FAILED FOR {myplace} =============')
@jit
def eval_gini(y_true, y_prob):
y_true = np.asarray(y_true)
 y_true = y_true[np.argsort(y_prob)]
ntrue = 0
gini = 0
delta = 0
n = len(y_true)
 for i in range(n-1, -1, -1):
y_i = y_true[i]
ntrue += y_i
gini += y_i * delta
delta += 1 - y_i
gini = 1 - 2 * gini /(ntrue *(n - ntrue))
return gini<load_from_csv>
myplace = 'US_Virgin Islands'
dat = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Place','Days_Since_Ten_Cases','ConfirmedCases']].dropna()
train_df = pd.read_csv('.. /input/train.csv', na_values="-1")
 test_df = pd.read_csv('.. /input/test.csv', na_values="-1" )<data_type_conversions>
tt.loc[tt['Place'].isin(us_states)].groupby('Place')['Fatalities_Pred1'].max().sum()
id_test = test_df['id'].values
id_train = train_df['id'].values
train_df = train_df.fillna(999)
 test_df = test_df.fillna(999)
 col_to_drop = train_df.columns[train_df.columns.str.startswith('ps_calc_')]
train_df = train_df.drop(col_to_drop, axis=1)
 test_df = test_df.drop(col_to_drop, axis=1)
 for c in train_df.select_dtypes(include=['float64'] ).columns:
train_df[c]=train_df[c].astype(np.float32)
 test_df[c]=test_df[c].astype(np.float32)
 for c in train_df.select_dtypes(include=['int64'] ).columns[2:]:
train_df[c]=train_df[c].astype(np.int8)
 test_df[c]=test_df[c].astype(np.int8)
 y = train_df['target']
X = train_df.drop(['target', 'id'], axis=1)
 y_valid_pred = 0*y
X_test = test_df.drop(['id'], axis=1)
 y_test_pred = 0<choose_model_class>
tt.loc[tt['Place'].isin(constant_fatal_places), 'ConfirmedCases_Pred1'] = tt.loc[tt['Place'].isin(constant_fatal_places)]['Place'].map(tt.loc[tt['Place'].isin(constant_fatal_places)].groupby('Place')['ConfirmedCases'].max())
 tt.loc[tt['Place'].isin(constant_fatal_places), 'Fatalities_Pred1'] = tt.loc[tt['Place'].isin(constant_fatal_places)]['Place'].map(tt.loc[tt['Place'].isin(constant_fatal_places)].groupby('Place')['Fatalities'].max() )
K = 5
kf = KFold(n_splits = K, random_state = 1, shuffle = True )<choose_model_class>
remaining_places = pd.DataFrame(tt.groupby('Place')['ConfirmedCases_Pred1'].max().isna() ).query('ConfirmedCases_Pred1' ).index.values
print(remaining_places)
 print(len(remaining_places))
model = CatBoostClassifier(
 learning_rate=LEARNING_RATE,
depth=6,
l2_leaf_reg = 14,
iterations = MAX_ROUNDS,
loss_function='Logloss'
)<train_model>
for myplace in remaining_places:
try:
fig, axs = plt.subplots(1, 2, figsize=(15, 3))
dat = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','ConfirmedCases_Log']].dropna()
X = dat['Days_Since_Ten_Cases']
y = dat['ConfirmedCases_Log']
y = y.cummax()
dat_all = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','ConfirmedCases_Log']]
X_pred = dat_all['Days_Since_Ten_Cases']
en = ElasticNet()
en.fit(X.values.reshape(-1, 1), y.values)
 preds = en.predict(X_pred.values.reshape(-1, 1))
tt.loc[(tt['Place'] == myplace)&(tt['Days_Since_Ten_Cases'] >= 0), 'ConfirmedCases_Log_Pred1'] = preds
tt.loc[(tt['Place'] == myplace), 'ConfirmedCases_Pred1'] = tt['ConfirmedCases_Log_Pred1'].apply(np.expm1)
 pop_myplace = tt.query('Place == @myplace')['Population_final'].values[0]
tt.loc[(tt['Place'] == myplace)&(tt['ConfirmedCases_Pred1'] >(0.05 * pop_myplace)) , 'ConfirmedCases_Pred1'] =(0.05 * pop_myplace)
 ax = tt.query('Place == @myplace' ).set_index('Date')[['ConfirmedCases','ConfirmedCases_Pred1']].plot(figsize=(15, 5), title=myplace, ax=axs[0])
 dat = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','Fatalities_Log']].dropna()
if len(dat)< 5:
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt.loc[(tt['Place'] == myplace)]['ConfirmedCases_Pred1'] * 0.0001
elif tt.query('Place == @myplace')['Fatalities'].max() < 5:
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt.loc[(tt['Place'] == myplace)]['ConfirmedCases_Pred1'] * 0.0001
else:
X = dat['Days_Since_Ten_Cases']
y = dat['Fatalities_Log']
y = y.cummax()
dat_all = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','Fatalities_Log']]
X_pred = dat_all['Days_Since_Ten_Cases']
en = ElasticNet()
en.fit(X.values.reshape(-1, 1), y.values)
 preds = en.predict(X_pred.values.reshape(-1, 1))
tt.loc[(tt['Place'] == myplace)&(tt['Days_Since_Ten_Cases'] >= 0), 'Fatalities_Log_Pred1'] = preds
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt['Fatalities_Log_Pred1'].apply(np.expm1)
 pop_myplace = tt.query('Place == @myplace')['Population_final'].values[0]
tt.loc[(tt['Place'] == myplace)&(tt['Fatalities_Pred1'] >(0.0001 * pop_myplace)) , 'Fatalities_Pred1'] =(0.0001 * pop_myplace)
 ax = tt.query('Place == @myplace' ).set_index('Date')[['Fatalities','Fatalities_Pred1']].plot(figsize=(15, 5), title=myplace, ax=axs[1])
 plt.show()
except:
print(f'============= FAILED FOR {myplace} =============')
for i,(train_index, test_index)in enumerate(kf.split(train_df)) :
y_train, y_valid = y.iloc[train_index], y.iloc[test_index]
X_train, X_valid = X.iloc[train_index,:], X.iloc[test_index,:]
print("
Fold ", i)
 if OPTIMIZE_ROUNDS:
fit_model = model.fit(X_train, y_train,
eval_set=[X_valid, y_valid],
use_best_model=True 
)
 print(" N trees = ", model.tree_count_)
 else:
fit_model = model.fit(X_train, y_train)
 pred = fit_model.predict_proba(X_valid)[:,1]
print(" Gini = ", eval_gini(y_valid, pred))
y_valid_pred.iloc[test_index] = pred
y_test_pred += fit_model.predict_proba(X_test)[:,1]
y_test_pred /= K
print("
Gini for full training set:")
 eval_gini(y, y_valid_pred )<save_to_csv>
tt['ConfirmedCases_Pred1'] = tt[['ConfirmedCases','ConfirmedCases_Pred1']].max(axis=1)
 tt['Fatalities_Pred1'] = tt[['Fatalities','Fatalities_Pred1']].max(axis=1)
 tt['ConfirmedCases_Pred1'] = tt['ConfirmedCases_Pred1'].fillna(0)
 tt['Fatalities_Pred1'] = tt['Fatalities_Pred1'].fillna(0)
 tt.loc[~tt['ConfirmedCases'].isna() , 'ConfirmedCases_Pred1'] = tt.loc[~tt['ConfirmedCases'].isna() ]['ConfirmedCases']
tt.loc[~tt['Fatalities'].isna() , 'Fatalities_Pred1'] = tt.loc[~tt['Fatalities'].isna() ]['Fatalities']
tt['ConfirmedCases_Pred1'] = tt.groupby('Place')['ConfirmedCases_Pred1'].transform('cummax')
 tt['Fatalities_Pred1'] = tt.groupby('Place')['Fatalities_Pred1'].transform('cummax' )
val = pd.DataFrame()
val['id'] = id_train
val['target'] = y_valid_pred.values
val.to_csv('cat_valid.csv', float_format='%.6f', index=False )<save_to_csv>
tt.groupby('Place')['Fatalities_Pred1'].max().sort_values()
sub = pd.DataFrame()
sub['id'] = id_test
sub['target'] = y_test_pred
sub.to_csv('cat_submit.csv', float_format='%.6f', index=False )<import_modules>
for myplace in ['Iran']:
fig, axs = plt.subplots(1, 2, figsize=(15, 3))
dat = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','ConfirmedCases']].dropna()
dat = dat.iloc[-10:]
X = dat['Days_Since_Ten_Cases']
y = dat['ConfirmedCases']
y = y.cummax()
dat_all = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','ConfirmedCases']]
X_pred = dat_all['Days_Since_Ten_Cases']
en = ElasticNet()
en.fit(X.values.reshape(-1, 1), y.values)
 preds = en.predict(X_pred.values.reshape(-1, 1))
tt.loc[(tt['Place'] == myplace)&(tt['Days_Since_Ten_Cases'] >= 0), 'ConfirmedCases_Pred1'] = preds
pop_myplace = tt.query('Place == @myplace')['Population_final'].values[0]
tt.loc[(tt['Place'] == myplace)&(tt['ConfirmedCases_Pred1'] >(0.1 * pop_myplace)) , 'ConfirmedCases_Pred1'] =(0.1 * pop_myplace)
 ax = tt.query('Place == @myplace' ).set_index('Date')[['ConfirmedCases','ConfirmedCases_Pred1']].plot(figsize=(15, 5), title=myplace, ax=axs[0])
 dat = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','Fatalities']].dropna()
dat = dat.iloc[-10:]
if len(dat)< 5:
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt.loc[(tt['Place'] == myplace)]['ConfirmedCases_Pred1'] * 0.001
elif tt.query('Place == @myplace')['Fatalities'].max() < 5:
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt.loc[(tt['Place'] == myplace)]['ConfirmedCases_Pred1'] * 0.001
else:
X = dat['Days_Since_Ten_Cases']
y = dat['Fatalities']
y = y.cummax()
dat_all = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','Fatalities']]
X_pred = dat_all['Days_Since_Ten_Cases']
en = ElasticNet()
en.fit(X.values.reshape(-1, 1), y.values)
 preds = en.predict(X_pred.values.reshape(-1, 1))
tt.loc[(tt['Place'] == myplace)&(tt['Days_Since_Ten_Cases'] >= 0), 'Fatalities_Pred1'] = preds
pop_myplace = tt.query('Place == @myplace')['Population_final'].values[0]
tt.loc[(tt['Place'] == myplace)&(tt['Fatalities_Pred1'] >(0.0005 * pop_myplace)) , 'Fatalities_Pred1'] =(0.0005 * pop_myplace)
 ax = tt.query('Place == @myplace' ).set_index('Date')[['Fatalities','Fatalities_Pred1']].plot(figsize=(15, 5), title=myplace, ax=axs[1])
 plt.show()
import matplotlib.pyplot as plt
from tqdm import tqdm_notebook as tqdm
import warnings
import sys
import gc
from scipy.stats import kurtosis
from scipy.stats import skew
from scipy.stats import kstat
from scipy.signal import hilbert
from scipy.signal import hann
from scipy.signal import convolve
from scipy.signal import welch
import scipy.signal as sg
from tsfresh.feature_extraction import feature_calculators
import nolds
import librosa
from imblearn.over_sampling import SMOTE
from catboost import CatBoostRegressor, Pool
import tensorflow
from keras.models import Sequential
from keras.layers import Dense, LSTM
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.preprocessing import StandardScaler, RobustScaler, MaxAbsScaler
from sklearn.svm import NuSVR
from sklearn.svm import SVR
from sklearn.manifold import TSNE
from sklearn.metrics import mean_absolute_error, SCORERS
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import LinearRegression, Lasso, MultiTaskLasso, ElasticNet, MultiTaskElasticNet, Ridge
from sklearn.neural_network import MLPRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor, BaggingRegressor<set_options>
tt['ConfirmedCases_Pred1'] = tt[['ConfirmedCases','ConfirmedCases_Pred1']].max(axis=1)
 tt['Fatalities_Pred1'] = tt[['Fatalities','Fatalities_Pred1']].max(axis=1)
 tt['ConfirmedCases_Pred1'] = tt['ConfirmedCases_Pred1'].fillna(0)
 tt['Fatalities_Pred1'] = tt['Fatalities_Pred1'].fillna(0)
 tt.loc[~tt['ConfirmedCases'].isna() , 'ConfirmedCases_Pred1'] = tt.loc[~tt['ConfirmedCases'].isna() ]['ConfirmedCases']
tt.loc[~tt['Fatalities'].isna() , 'Fatalities_Pred1'] = tt.loc[~tt['Fatalities'].isna() ]['Fatalities']
tt['ConfirmedCases_Pred1'] = tt.groupby('Place')['ConfirmedCases_Pred1'].transform('cummax')
 tt['Fatalities_Pred1'] = tt.groupby('Place')['Fatalities_Pred1'].transform('cummax' )
%matplotlib inline
%precision 4
warnings.filterwarnings('ignore')
 plt.style.use('ggplot')
 np.set_printoptions(suppress=True)
 pd.set_option("display.precision", 15 )<import_modules>
ss = pd.read_csv('.. /input/covid19-global-forecasting-week-2/submission.csv' )
<load_from_csv><EOS> 
mysub = tt.dropna(subset=['ForecastId'])[['ForecastId','ConfirmedCases_Pred1','Fatalities_Pred1']]
mysub['ForecastId'] = mysub['ForecastId'].astype('int')
 mysub = mysub.rename(columns={'ConfirmedCases_Pred1':'ConfirmedCases',
'Fatalities_Pred1': 'Fatalities'})
 mysub.to_csv('submission.csv', index=False )