Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<sort_values>
pd.set_option('max_columns', 500)
 pd.set_option('max_rows', 500)
 pd.options.display.float_format = '{:.2f}'.format
np.abs(X_data.corrwith(pd.Series(y_data)) ).sort_values(ascending=False ).head(20 )<set_options>
train = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 tt = pd.concat([train, test], sort=False)
 tt = train.merge(test, on=['Province_State','Country_Region','Date'], how='outer')
 def name_place(x):
try:
x_new = x['Country_Region'] + "_" + x['Province_State']
except:
x_new = x['Country_Region']
return x_new
tt['Place'] = tt.apply(lambda x: name_place(x), axis=1)
 tt['Date'] = pd.to_datetime(tt['Date'])
 tt['doy'] = tt['Date'].dt.dayofyear
tt['dow'] = tt['Date'].dt.dayofweek
tt['hasProvidence'] = ~tt['Province_State'].isna()
country_meta = pd.read_csv('.. /input/covid19-forecasting-metadata/region_metadata.csv')
 tt = tt.merge(country_meta, how='left')
 country_date_meta = pd.read_csv('.. /input/covid19-forecasting-metadata/region_date_metadata.csv')
 tt['HasFatality'] = tt.groupby('Place')['Fatalities'].transform(lambda x: x.max() > 0)
 tt['HasCases'] = tt.groupby('Place')['ConfirmedCases'].transform(lambda x: x.max() > 0)
 first_case_date = tt.query('ConfirmedCases >= 1' ).groupby('Place')['Date'].min().to_dict()
ten_case_date = tt.query('ConfirmedCases >= 10' ).groupby('Place')['Date'].min().to_dict()
hundred_case_date = tt.query('ConfirmedCases >= 100' ).groupby('Place')['Date'].min().to_dict()
first_fatal_date = tt.query('Fatalities >= 1' ).groupby('Place')['Date'].min().to_dict()
ten_fatal_date = tt.query('Fatalities >= 10' ).groupby('Place')['Date'].min().to_dict()
hundred_fatal_date = tt.query('Fatalities >= 100' ).groupby('Place')['Date'].min().to_dict()
tt['First_Case_Date'] = tt['Place'].map(first_case_date)
 tt['Ten_Case_Date'] = tt['Place'].map(ten_case_date)
 tt['Hundred_Case_Date'] = tt['Place'].map(hundred_case_date)
 tt['First_Fatal_Date'] = tt['Place'].map(first_fatal_date)
 tt['Ten_Fatal_Date'] = tt['Place'].map(ten_fatal_date)
 tt['Hundred_Fatal_Date'] = tt['Place'].map(hundred_fatal_date)
 tt['Days_Since_First_Case'] =(tt['Date'] - tt['First_Case_Date'] ).dt.days
tt['Days_Since_Ten_Cases'] =(tt['Date'] - tt['Ten_Case_Date'] ).dt.days
tt['Days_Since_Hundred_Cases'] =(tt['Date'] - tt['Hundred_Case_Date'] ).dt.days
tt['Days_Since_First_Fatal'] =(tt['Date'] - tt['First_Fatal_Date'] ).dt.days
tt['Days_Since_Ten_Fatal'] =(tt['Date'] - tt['Ten_Fatal_Date'] ).dt.days
tt['Days_Since_Hundred_Fatal'] =(tt['Date'] - tt['Hundred_Fatal_Date'] ).dt.days
smoking = pd.read_csv(".. /input/smokingstats/share-of-adults-who-smoke.csv")
 smoking = smoking.rename(columns={'Smoking prevalence, total(ages 15+ )(% of adults)': 'Smoking_Rate'})
 smoking_dict = smoking.groupby('Entity')['Year'].max().to_dict()
smoking['LastYear'] = smoking['Entity'].map(smoking_dict)
 smoking = smoking.query('Year == LastYear' ).reset_index()
smoking['Entity'] = smoking['Entity'].str.replace('United States', 'US')
 tt = tt.merge(smoking[['Entity','Smoking_Rate']],
left_on='Country_Region',
right_on='Entity',
how='left',
validate='m:1')\ 
.drop('Entity', axis=1)
 country_info = pd.read_csv('.. /input/countryinfo/covid19countryinfo.csv')
 tt = tt.merge(country_info, left_on=['Country_Region','Province_State'],
right_on=['country','region'],
how='left',
validate='m:1')
 us_state_info = pd.read_html('https://simple.wikipedia.org/wiki/List_of_U.S._states_by_population')[0] \
[['State','Population estimate, July 1, 2019[2]']] \ 
.rename(columns={'Population estimate, July 1, 2019[2]' : 'Population'})
 tt = tt.merge(us_state_info[['State','Population']],
left_on='Province_State',
right_on='State',
how='left')
 tt['pop'] = pd.to_numeric(tt['pop'].str.replace(',',''))
tt['pop'] = tt['pop'].fillna(tt['Population'])
 tt['pop'] = pd.to_numeric(tt['pop'])
 tt['pop_diff'] = tt['pop'] - tt['Population']
tt['Population_final'] = tt['Population']
tt.loc[~tt['hasProvidence'], 'Population_final'] = tt.loc[~tt['hasProvidence']]['pop']
tt['Confirmed_Cases_Diff'] = tt.groupby('Place')['ConfirmedCases'].diff()
tt['Fatailities_Diff'] = tt.groupby('Place')['Fatalities'].diff()
max_date = tt.dropna(subset=['ConfirmedCases'])['Date'].max()
tt['gdp2019'] = pd.to_numeric(tt['gdp2019'].str.replace(',',''))
gc.collect()<split>
pop_dict = {'Angola': int(29.78 * 10**6),
'Australia_Australian Capital Territory': 423_800,
'Australia_New South Wales': int(7.544 * 10**6),
'Australia_Northern Territory': 244_300,
'Australia_Queensland' : int(5.071 * 10**6),
'Australia_South Australia' : int(1.677 * 10**6),
'Australia_Tasmania': 515_000,
'Australia_Victoria': int(6.359 * 10**6),
'Australia_Western Australia': int(2.589 * 10**6),
'Brazil': int(209.3 * 10**6),
'Canada_Alberta' : int(4.371 * 10**6),
'Canada_British Columbia' : int(5.071 * 10**6),
'Canada_Manitoba' : int(1.369 * 10**6),
'Canada_New Brunswick' : 776_827,
'Canada_Newfoundland and Labrador' : 521_542,
'Canada_Nova Scotia' : 971_395,
'Canada_Ontario' : int(14.57 * 10**6),
'Canada_Prince Edward Island' : 156_947,
'Canada_Quebec' : int(8.485 * 10**6),
'Canada_Saskatchewan': int(1.174 * 10**6),
'China_Anhui': int(62 * 10**6),
'China_Beijing': int(21.54 * 10**6),
'China_Chongqing': int(30.48 * 10**6),
'China_Fujian' : int(38.56 * 10**6),
'China_Gansu' : int(25.58 * 10**6),
'China_Guangdong' : int(113.46 * 10**6),
'China_Guangxi' : int(48.38 * 10**6),
'China_Guizhou' : int(34.75 * 10**6),
'China_Hainan' : int(9.258 * 10**6),
'China_Hebei' : int(74.7 * 10**6),
'China_Heilongjiang' : int(38.31 * 10**6),
'China_Henan' : int(94 * 10**6),
'China_Hong Kong' : int(7.392 * 10**6),
'China_Hubei' : int(58.5 * 10**6),
'China_Hunan' : int(67.37 * 10**6),
'China_Inner Mongolia' : int(24.71 * 10**6),
'China_Jiangsu' : int(80.4 * 10**6),
'China_Jiangxi' : int(45.2 * 10**6),
'China_Jilin' : int(27.3 * 10**6),
'China_Liaoning' : int(43.9 * 10**6),
'China_Macau' : 622_567,
'China_Ningxia' : int(6.301 * 10**6),
'China_Qinghai' : int(5.627 * 10**6),
'China_Shaanxi' : int(37.33 * 10**6),
'China_Shandong' : int(92.48 * 10**6),
'China_Shanghai' : int(24.28 * 10**6),
'China_Shanxi' : int(36.5 * 10**6),
'China_Sichuan' : int(81.1 * 10**6),
'China_Tianjin' : int(15 * 10**6),
'China_Tibet' : int(3.18 * 10**6),
'China_Xinjiang' : int(21.81 * 10**6),
'China_Yunnan' : int(45.97 * 10**6),
'China_Zhejiang' : int(57.37 * 10**6),
'Denmark_Faroe Islands' : 51_783,
'Denmark_Greenland' : 56_171,
'France_French Guiana' : 290_691,
'France_French Polynesia' : 283_007,
'France_Guadeloupe' : 395_700,
'France_Martinique' : 376_480,
'France_Mayotte' : 270_372,
'France_New Caledonia' : 99_926,
'France_Reunion' : 859_959,
'France_Saint Barthelemy' : 9_131,
'France_St Martin' : 32_125,
'Netherlands_Aruba' : 105_264,
'Netherlands_Curacao' : 161_014,
'Netherlands_Sint Maarten' : 41_109,
'Papua New Guinea' : int(8.251 * 10**6),
'US_Guam' : 164_229,
'US_Virgin Islands' : 107_268,
'United Kingdom_Bermuda' : 65_441,
'United Kingdom_Cayman Islands' : 61_559,
'United Kingdom_Channel Islands' : 170_499,
'United Kingdom_Gibraltar' : 34_571,
'United Kingdom_Isle of Man' : 84_287,
'United Kingdom_Montserrat' : 4_922
}
tt['Population_final'] = tt['Population_final'].fillna(tt['Place'].map(pop_dict))
X_train, X_test, y_train, y_test = train_test_split(
 X_data, y_data,
test_size=0.2,
shuffle=True, random_state=42)
 X_train, X_valid, y_train, y_valid = train_test_split(
 X_train, y_train,
test_size=0.2,
shuffle=True, random_state=42 
)
 print("Training on", X_train.shape[0], "rows")
 print("Validating on", X_valid.shape[0], "rows")
 print("Testing on", X_test.shape[0], "rows")
 f,(ax1, ax2, ax3)= plt.subplots(1, 3, sharey=True)
 f.suptitle('Histogram of time_to_failure')
 ax1.hist(y_train, bins=17)
 ax1.set_title("train")
 ax2.hist(y_valid, bins=17)
 ax2.set_title("test")
 ax3.hist(y_test, bins=17)
 ax3.set_title("validation")
 plt.show()<normalization>
tt.loc[tt['Place'] == 'Diamond Princess', 'Population final'] = 2_670
scaler = StandardScaler()
scaler.fit(X_train)
 X_train = scaler.transform(X_train)
 X_test = scaler.transform(X_test)
 X_valid = scaler.transform(X_valid)
 print("Data scaled" )<train_model>
tt['ConfirmedCases_Log'] = tt['ConfirmedCases'].apply(np.log1p)
 tt['Fatalities_Log'] = tt['Fatalities'].apply(np.log1p )
dataPool = Pool(X_train, y_train)
 modelCatBoost = CatBoostRegressor(iterations=1000,
loss_function='MAE',
learning_rate=0.1,
depth=6,
l2_leaf_reg=3,
border_count=128,
use_best_model=True,
silent=True)
 modelCatBoost.fit(dataPool, eval_set=Pool(X_valid, y_valid), plot=True)
 y_pred = modelCatBoost.predict(X_test)
 score = plotAndScore(y_pred, y_test, "Catboost" )<normalization>
tt['Population_final'] = tt['Population_final'].astype('int')
 tt['Cases_Per_100kPop'] =(tt['ConfirmedCases'] / tt['Population_final'])* 100000
tt['Fatalities_Per_100kPop'] =(tt['Fatalities'] / tt['Population_final'])* 100000
tt['Cases_Percent_Pop'] =(( tt['ConfirmedCases'] / tt['Population_final'])* 100)
 tt['Fatalities_Percent_Pop'] =(( tt['Fatalities'] / tt['Population_final'])* 100)
 tt['Cases_Log_Percent_Pop'] =(( tt['ConfirmedCases'] / tt['Population_final'])* 100 ).apply(np.log1p)
 tt['Fatalities_Log_Percent_Pop'] =(( tt['Fatalities'] / tt['Population_final'])* 100 ).apply(np.log1p)
 tt['Max_Confirmed_Cases'] = tt.groupby('Place')['ConfirmedCases'].transform(max)
 tt['Max_Fatalities'] = tt.groupby('Place')['Fatalities'].transform(max)
 tt['Max_Cases_Per_100kPop'] = tt.groupby('Place')['Cases_Per_100kPop'].transform(max)
 tt['Max_Fatalities_Per_100kPop'] = tt.groupby('Place')['Fatalities_Per_100kPop'].transform(max )
params = {}
params["iterations"] = 500
params["loss_function"] = 'MAE'
params["learning_rate"] = 0.2
params["depth"] = 6
params["verbose"] = 0
cv_data = cv(pool=Pool(StandardScaler().fit_transform(X_data), y_data),
params=params,
fold_count=5,
inverted=False,
shuffle=True,
stratified=False,
as_pandas=True,
plot=True,
verbose=False)
 min_test_mean = cv_data["test-MAE-mean"].min()
print("best MAE :",min_test_mean )<choose_model_class>
tt.query('Date == @max_date')[['Place','Max_Cases_Per_100kPop',
'Max_Fatalities_Per_100kPop','Max_Confirmed_Cases',
'Population_final',
'Days_Since_First_Case',
'Confirmed_Cases_Diff']] \ 
.drop_duplicates() \ 
.sort_values('Max_Cases_Per_100kPop', ascending=False )
models = {
"CatBoostRegressor" : CatBoostRegressor(iterations=300, loss_function='MAE', learning_rate=0.2, depth=6, verbose=0),
"NuSVR" : NuSVR(kernel="rbf", C=1),
"SVR" : SVR(kernel="rbf", C=1),
"DecisionTreeRegressor" : DecisionTreeRegressor() ,
"RandomForestRegressor" : RandomForestRegressor(max_depth=6, n_estimators=200),
"GradientBoostingRegressor" : GradientBoostingRegressor() ,
"KNeighborsRegressor(n_neighbors=50)" : KNeighborsRegressor(n_neighbors=50),
'LinearRegression': LinearRegression() ,
'Ridge_1': Ridge(alpha=0.001),
'Lasso_1': Lasso(alpha=0.001),
'ElasticNet_1': ElasticNet(alpha=0.001, l1_ratio=0),
'ElasticNet_2': ElasticNet(alpha=0.001, l1_ratio=0.25),
'ElasticNet_3': ElasticNet(alpha=0.001, l1_ratio=0.33),
'ElasticNet_4': ElasticNet(alpha=0.001, l1_ratio=0.5),
'ElasticNet_5': ElasticNet(alpha=0.001, l1_ratio=0.66),
'ElasticNet_6': ElasticNet(alpha=0.001, l1_ratio=0.75),
'ElasticNet_7': ElasticNet(alpha=0.001, l1_ratio=1),
}<load_from_csv>
tt['Past_7Days_ConfirmedCases_Std'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200324' ).groupby('Place')['ConfirmedCases'].std().to_dict())
 tt['Past_7Days_Fatalities_Std'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200324' ).groupby('Place')['Fatalities'].std().to_dict())
 tt['Past_7Days_ConfirmedCases_Min'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200324' ).groupby('Place')['ConfirmedCases'].min().to_dict())
 tt['Past_7Days_Fatalities_Min'] = tt['Place'].map(tt.dropna(subset=['Fatalities'] ).query('Date >= 20200324' ).groupby('Place')['Fatalities'].min().to_dict())
 tt['Past_7Days_ConfirmedCases_Max'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200324' ).groupby('Place')['ConfirmedCases'].max().to_dict())
 tt['Past_7Days_Fatalities_Max'] = tt['Place'].map(tt.dropna(subset=['Fatalities'] ).query('Date >= 20200324' ).groupby('Place')['Fatalities'].max().to_dict())
 tt['Past_7Days_Confirmed_Change_of_Total'] =(tt['Past_7Days_ConfirmedCases_Max'] - tt['Past_7Days_ConfirmedCases_Min'])/(tt['Past_7Days_ConfirmedCases_Max'])
 tt['Past_7Days_Fatalities_Change_of_Total'] =(tt['Past_7Days_Fatalities_Max'] - tt['Past_7Days_Fatalities_Min'])/(tt['Past_7Days_Fatalities_Max'] )
submission = pd.read_csv('.. /input/sample_submission.csv', index_col='seg_id')
 X_data_submission = pd.DataFrame(dtype=np.float64, index=submission.index)
 plot_data_submission = pd.Series()
for seg_id in tqdm(X_data_submission.index):
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 plot_data_submission.append(seg[::50])
 x = seg['acoustic_data']
extract_features(X_data_submission, seg_id, x)
<predict_on_test>
tt['Past_21Days_ConfirmedCases_Std'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200310' ).groupby('Place')['ConfirmedCases'].std().to_dict())
 tt['Past_21Days_Fatalities_Std'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200310' ).groupby('Place')['Fatalities'].std().to_dict())
 tt['Past_21Days_ConfirmedCases_Min'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200310' ).groupby('Place')['ConfirmedCases'].min().to_dict())
 tt['Past_21Days_Fatalities_Min'] = tt['Place'].map(tt.dropna(subset=['Fatalities'] ).query('Date >= 20200324' ).groupby('Place')['Fatalities'].min().to_dict())
 tt['Past_21Days_ConfirmedCases_Max'] = tt['Place'].map(tt.dropna(subset=['ConfirmedCases'] ).query('Date >= 20200310' ).groupby('Place')['ConfirmedCases'].max().to_dict())
 tt['Past_21Days_Fatalities_Max'] = tt['Place'].map(tt.dropna(subset=['Fatalities'] ).query('Date >= 20200310' ).groupby('Place')['Fatalities'].max().to_dict())
 tt['Past_21Days_Confirmed_Change_of_Total'] =(tt['Past_21Days_ConfirmedCases_Max'] - tt['Past_21Days_ConfirmedCases_Min'])/(tt['Past_21Days_ConfirmedCases_Max'])
 tt['Past_21Days_Fatalities_Change_of_Total'] =(tt['Past_21Days_Fatalities_Max'] - tt['Past_21Days_Fatalities_Min'])/(tt['Past_21Days_Fatalities_Max'])
 tt['Past_7Days_Fatalities_Change_of_Total'] = tt['Past_7Days_Fatalities_Change_of_Total'].fillna(0)
 tt['Past_21Days_Fatalities_Change_of_Total'] = tt['Past_21Days_Fatalities_Change_of_Total'].fillna(0 )
X_data_submission = scaler.transform(X_data_submission)
 submission['time_to_failure'] = [x*1.05 for x in modelCatBoost.predict(X_data_submission)]<load_from_csv>
tt['Date_7Days_Since_First_Case'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Case'] == 7] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_14Days_Since_First_Case'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Case'] == 14] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_21Days_Since_First_Case'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Case'] == 21] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_28Days_Since_First_Case'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Case'] == 28] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_35Days_Since_First_Case'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Case'] == 35] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_60Days_Since_First_Case'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Case'] == 60] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_7Days_Since_Ten_Cases'] = tt['Place'].map(tt.loc[tt['Days_Since_Ten_Cases'] == 7] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_14Days_Since_Ten_Cases'] = tt['Place'].map(tt.loc[tt['Days_Since_Ten_Cases'] == 14] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_21Days_Since_Ten_Cases'] = tt['Place'].map(tt.loc[tt['Days_Since_Ten_Cases'] == 21] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_28Days_Since_Ten_Cases'] = tt['Place'].map(tt.loc[tt['Days_Since_Ten_Cases'] == 28] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_35Days_Since_Ten_Cases'] = tt['Place'].map(tt.loc[tt['Days_Since_Ten_Cases'] == 35] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_60Days_Since_Ten_Cases'] = tt['Place'].map(tt.loc[tt['Days_Since_Ten_Cases'] == 60] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_7Days_Since_First_Fatal'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Fatal'] == 7] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_14Days_Since_First_Fatal'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Fatal'] == 14] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_21Days_Since_First_Fatal'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Fatal'] == 21] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_28Days_Since_First_Fatal'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Fatal'] == 28] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_35Days_Since_First_Fatal'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Fatal'] == 35] \ 
.set_index('Place')['Date'] \ 
.to_dict())
 tt['Date_60Days_Since_First_Fatal'] = tt['Place'].map(tt.loc[tt['Days_Since_First_Fatal'] == 60] \ 
.set_index('Place')['Date'] \ 
.to_dict() )
acoustic_data_small = []
for seg_id in tqdm(submission.index):
seg = pd.read_csv('.. /input/test/' + seg_id + '.csv')
 acoustic_data_small.append(seg['acoustic_data'][::50])
<drop_column>
tt['CC_7D_1stCase'] = tt.loc[tt['Date_7Days_Since_First_Case'] == tt['Date']]['ConfirmedCases']
tt['CC_14D_1stCase'] = tt.loc[tt['Date_14Days_Since_First_Case'] == tt['Date']]['ConfirmedCases']
tt['CC_21D_1stCase'] = tt.loc[tt['Date_21Days_Since_First_Case'] == tt['Date']]['ConfirmedCases']
tt['CC_28D_1stCase'] = tt.loc[tt['Date_28Days_Since_First_Case'] == tt['Date']]['ConfirmedCases']
tt['CC_35D_1stCase'] = tt.loc[tt['Date_35Days_Since_First_Case'] == tt['Date']]['ConfirmedCases']
tt['CC_60D_1stCase'] = tt.loc[tt['Date_60Days_Since_First_Case'] == tt['Date']]['ConfirmedCases']
tt['F_7D_1stCase'] = tt.loc[tt['Date_7Days_Since_First_Case'] == tt['Date']]['Fatalities']
tt['F_14D_1stCase'] = tt.loc[tt['Date_14Days_Since_First_Case'] == tt['Date']]['Fatalities']
tt['F_21D_1stCase'] = tt.loc[tt['Date_21Days_Since_First_Case'] == tt['Date']]['Fatalities']
tt['F_28D_1stCase'] = tt.loc[tt['Date_28Days_Since_First_Case'] == tt['Date']]['Fatalities']
tt['F_35D_1stCase'] = tt.loc[tt['Date_35Days_Since_First_Case'] == tt['Date']]['Fatalities']
tt['F_60D_1stCase'] = tt.loc[tt['Date_60Days_Since_First_Case'] == tt['Date']]['Fatalities']
tt['CC_7D_10Case'] = tt.loc[tt['Date_7Days_Since_Ten_Cases'] == tt['Date']]['ConfirmedCases']
tt['CC_14D_10Case'] = tt.loc[tt['Date_14Days_Since_Ten_Cases'] == tt['Date']]['ConfirmedCases']
tt['CC_21D_10Case'] = tt.loc[tt['Date_21Days_Since_Ten_Cases'] == tt['Date']]['ConfirmedCases']
tt['CC_28D_10Case'] = tt.loc[tt['Date_28Days_Since_Ten_Cases'] == tt['Date']]['ConfirmedCases']
tt['CC_35D_10Case'] = tt.loc[tt['Date_35Days_Since_Ten_Cases'] == tt['Date']]['ConfirmedCases']
tt['CC_60D_10Case'] = tt.loc[tt['Date_60Days_Since_Ten_Cases'] == tt['Date']]['ConfirmedCases']
tt['F_7D_10Case'] = tt.loc[tt['Date_7Days_Since_Ten_Cases'] == tt['Date']]['Fatalities']
tt['F_14D_10Case'] = tt.loc[tt['Date_14Days_Since_Ten_Cases'] == tt['Date']]['Fatalities']
tt['F_21D_10Case'] = tt.loc[tt['Date_21Days_Since_Ten_Cases'] == tt['Date']]['Fatalities']
tt['F_28D_10Case'] = tt.loc[tt['Date_28Days_Since_Ten_Cases'] == tt['Date']]['Fatalities']
tt['F_35D_10Case'] = tt.loc[tt['Date_35Days_Since_Ten_Cases'] == tt['Date']]['Fatalities']
tt['F_60D_10Case'] = tt.loc[tt['Date_60Days_Since_Ten_Cases'] == tt['Date']]['Fatalities']
tt['CC_7D_1Fatal'] = tt.loc[tt['Date_7Days_Since_First_Fatal'] == tt['Date']]['ConfirmedCases']
tt['CC_14D_1Fatal'] = tt.loc[tt['Date_14Days_Since_First_Fatal'] == tt['Date']]['ConfirmedCases']
tt['CC_21D_1Fatal'] = tt.loc[tt['Date_21Days_Since_First_Fatal'] == tt['Date']]['ConfirmedCases']
tt['CC_28D_1Fatal'] = tt.loc[tt['Date_28Days_Since_First_Fatal'] == tt['Date']]['ConfirmedCases']
tt['CC_35D_1Fatal'] = tt.loc[tt['Date_35Days_Since_First_Fatal'] == tt['Date']]['ConfirmedCases']
tt['CC_60D_1Fatal'] = tt.loc[tt['Date_60Days_Since_First_Fatal'] == tt['Date']]['ConfirmedCases']
tt['F_7D_1Fatal'] = tt.loc[tt['Date_7Days_Since_First_Fatal'] == tt['Date']]['Fatalities']
tt['F_14D_1Fatal'] = tt.loc[tt['Date_14Days_Since_First_Fatal'] == tt['Date']]['Fatalities']
tt['F_21D_1Fatal'] = tt.loc[tt['Date_21Days_Since_First_Fatal'] == tt['Date']]['Fatalities']
tt['F_28D_1Fatal'] = tt.loc[tt['Date_28Days_Since_First_Fatal'] == tt['Date']]['Fatalities']
tt['F_35D_1Fatal'] = tt.loc[tt['Date_35Days_Since_First_Fatal'] == tt['Date']]['Fatalities']
tt['F_60D_1Fatal'] = tt.loc[tt['Date_60Days_Since_First_Fatal'] == tt['Date']]['Fatalities']
del acoustic_data_small
del time_to_failure_small
gc.collect()<save_to_csv>
latest_summary_stats = tt.query('Date == @max_date')\
[['Country_Region',
'Place',
'Max_Cases_Per_100kPop',
'Max_Fatalities_Per_100kPop',
'Max_Confirmed_Cases',
'Population_final',
'Days_Since_First_Case',
'Days_Since_Ten_Cases']] \ 
.drop_duplicates()
submission.to_csv('submission.csv')
 print("Submission data saved to csv." )<load_from_csv>
SEED = 42
params = {'num_leaves': 8,
'min_data_in_leaf': 5,
'objective': 'regression',
'max_depth': 8,
'learning_rate': 0.02,
'bagging_freq': 5,
'bagging_fraction': 0.8,
'feature_fraction': 0.8201,
'bagging_seed': SEED,
'reg_alpha': 1,
'reg_lambda': 4.9847051755586085,
'random_state': SEED,
'metric': 'mse',
'verbosity': 100,
'n_estimators' : 500,
'min_gain_to_split': 0.02,
'min_child_weight': 5,
'num_threads': 6,
}
xgb = pd.read_csv(".. /input/final-subs/xgb_final.csv")
 print(xgb.shape)
 lgb = pd.read_csv(".. /input/lgbm-tunning-bayes-search/clean_submission.csv")
 print(lgb.shape)
 cat = pd.read_csv(".. /input/final-subs/cat_boost.csv")
 print(cat.shape)
 xgb.head()<rename_columns>
pd.set_option('max_columns', 500)
 try:
tt['healthexp'] = pd.to_numeric(tt['healthexp'].str.replace(',',''))
except:
pass
train = tt.loc[tt['Date'] < '25-March-2020']
test = tt.loc[tt['Date'] >= '25-March-2020']
comb = cat
comb.rename(columns={'time_to_failure':'cat'},inplace=True)
 comb.head()<feature_engineering>
FEATURES = ['doy', 'dow', 'hasProvidence',
'lat', 'lon', 'Days_Since_First_Case', 'Days_Since_Ten_Cases',
'Days_Since_Hundred_Cases', 'Days_Since_First_Fatal',
'Days_Since_Ten_Fatal', 'Days_Since_Hundred_Fatal', 'Smoking_Rate',
'Population_final',
'hospibed', 'smokers', 'sex0', 'sex14', 'sex25', 'sex54', 'sex64',
'sex65plus', 'sexratio', 'lung', 'femalelung', 'malelung', 'gdp2019',
'healthexp', 'healthperpop', 'fertility',
'CC_7D_1stCase',
'CC_14D_1stCase',
'CC_21D_1stCase',
'CC_28D_1stCase',
'CC_35D_1stCase',
'CC_60D_1stCase',
'F_7D_1stCase',
'F_14D_1stCase',
'F_21D_1stCase',
'F_28D_1stCase',
'F_35D_1stCase',
'F_60D_1stCase',
'CC_7D_10Case',
'CC_14D_10Case',
'CC_21D_10Case',
'CC_28D_10Case',
'CC_35D_10Case',
'CC_60D_10Case',
'F_7D_10Case',
'F_14D_10Case',
'F_21D_10Case',
'F_28D_10Case',
'F_35D_10Case',
'F_60D_10Case',
'CC_7D_1Fatal',
'CC_14D_1Fatal',
'CC_21D_1Fatal',
'CC_28D_1Fatal',
'CC_35D_1Fatal',
'CC_60D_1Fatal',
'F_7D_1Fatal',
'F_14D_1Fatal',
'F_21D_1Fatal',
'F_28D_1Fatal',
'F_35D_1Fatal',
'F_60D_1Fatal'
]
TARGET = ['ConfirmedCases_Log']
comb['lgb'] = lgb.time_to_failure
comb.head()<feature_engineering>
reg = lgb.LGBMRegressor(**params)
 reg.fit(train[FEATURES], train[TARGET] )
comb['xgb'] = xgb.time_to_failure
comb.head()<feature_engineering>
test['ConfirmedCases_Log_Pred'] = reg.predict(test[FEATURES])
 test['ConfirmedCases_pred'] = test['ConfirmedCases_Log_Pred'].apply(np.expm1 )
comb['time_to_failure'] = comb.mean(numeric_only=True, axis=1)
 comb.head()<prepare_output>
from sklearn.linear_model import LinearRegression, ElasticNet
sub = comb.loc[:,['seg_id','time_to_failure']]
sub.head()<save_to_csv>
myplace = "US_Florida"
tt.query('Place == @myplace and Days_Since_First_Fatal >= 0')[['Days_Since_First_Fatal','Fatalities_Log']]
sub.to_csv('submission.csv',index=False )<set_options>
for myplace in us_states:
try:
fig, axs = plt.subplots(1, 2, figsize=(15, 3))
dat = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','ConfirmedCases_Log']].dropna()
X = dat['Days_Since_Ten_Cases']
y = dat['ConfirmedCases_Log']
y = y.cummax()
dat_all = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','ConfirmedCases_Log']]
X_pred = dat_all['Days_Since_Ten_Cases']
en = ElasticNet()
en.fit(X.values.reshape(-1, 1), y.values)
 preds = en.predict(X_pred.values.reshape(-1, 1))
tt.loc[(tt['Place'] == myplace)&(tt['Days_Since_Ten_Cases'] >= 0), 'ConfirmedCases_Log_Pred1'] = preds
tt.loc[(tt['Place'] == myplace), 'ConfirmedCases_Pred1'] = tt['ConfirmedCases_Log_Pred1'].apply(np.expm1)
 pop_myplace = tt.query('Place == @myplace')['Population_final'].values[0]
tt.loc[(tt['Place'] == myplace)&(tt['ConfirmedCases_Pred1'] >(0.1 * pop_myplace)) , 'ConfirmedCases_Pred1'] =(0.1 * pop_myplace)
 ax = tt.query('Place == @myplace' ).set_index('Date')[['ConfirmedCases','ConfirmedCases_Pred1']].plot(figsize=(15, 5), title=myplace, ax=axs[0])
 dat = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','Fatalities_Log']].dropna()
if len(dat)< 5:
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt.loc[(tt['Place'] == myplace)]['ConfirmedCases_Pred1'] * 0.001
elif tt.query('Place == @myplace')['Fatalities'].max() < 5:
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt.loc[(tt['Place'] == myplace)]['ConfirmedCases_Pred1'] * 0.001
else:
X = dat['Days_Since_Ten_Cases']
y = dat['Fatalities_Log']
y = y.cummax()
dat_all = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','Fatalities_Log']]
X_pred = dat_all['Days_Since_Ten_Cases']
en = ElasticNet()
en.fit(X.values.reshape(-1, 1), y.values)
 preds = en.predict(X_pred.values.reshape(-1, 1))
tt.loc[(tt['Place'] == myplace)&(tt['Days_Since_Ten_Cases'] >= 0), 'Fatalities_Log_Pred1'] = preds
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt['Fatalities_Log_Pred1'].apply(np.expm1)
 pop_myplace = tt.query('Place == @myplace')['Population_final'].values[0]
tt.loc[(tt['Place'] == myplace)&(tt['Fatalities_Pred1'] >(0.0005 * pop_myplace)) , 'Fatalities_Pred1'] =(0.0005 * pop_myplace)
 ax = tt.query('Place == @myplace' ).set_index('Date')[['Fatalities','Fatalities_Pred1']].plot(figsize=(15, 5), title=myplace, ax=axs[1])
 plt.show()
except:
print(f'============= FAILED FOR {myplace} =============')
warnings.filterwarnings("ignore")
<load_from_csv>
myplace = 'US_Virgin Islands'
dat = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Place','Days_Since_Ten_Cases','ConfirmedCases']].dropna()
scaled_train_X = pd.read_csv('.. /input/all-features/train_X.csv')
 scaled_test_X = pd.read_csv('.. /input/all-features/test_X.csv')
 train_y = pd.read_csv('.. /input/all-features/train_y.csv' )<load_from_csv>
tt.loc[tt['Place'].isin(us_states)].groupby('Place')['Fatalities_Pred1'].max().sum()
submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv', index_col='seg_id')
 submission.shape<choose_model_class>
tt.loc[tt['Place'].isin(constant_fatal_places), 'ConfirmedCases_Pred1'] = tt.loc[tt['Place'].isin(constant_fatal_places)]['Place'].map(tt.loc[tt['Place'].isin(constant_fatal_places)].groupby('Place')['ConfirmedCases'].max())
 tt.loc[tt['Place'].isin(constant_fatal_places), 'Fatalities_Pred1'] = tt.loc[tt['Place'].isin(constant_fatal_places)]['Place'].map(tt.loc[tt['Place'].isin(constant_fatal_places)].groupby('Place')['Fatalities'].max() )
mse_scoring = make_scorer(mean_squared_error, greater_is_better=False)
 k_fold = KFold(n_splits=7, shuffle=True, random_state=13 )<compute_train_metric>
remaining_places = pd.DataFrame(tt.groupby('Place')['ConfirmedCases_Pred1'].max().isna() ).query('ConfirmedCases_Pred1' ).index.values
print(remaining_places)
 print(len(remaining_places))
def make_objective(model, X, y, space, cv, scoring):
@use_named_args(space)
 def objective(**params):
model.set_params(**params)
 return -np.mean(cross_val_score(model,
X, y,
cv=cv,
n_jobs=-1,
scoring=scoring))
return objective<init_hyperparams>
for myplace in remaining_places:
try:
fig, axs = plt.subplots(1, 2, figsize=(15, 3))
dat = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','ConfirmedCases_Log']].dropna()
X = dat['Days_Since_Ten_Cases']
y = dat['ConfirmedCases_Log']
y = y.cummax()
dat_all = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','ConfirmedCases_Log']]
X_pred = dat_all['Days_Since_Ten_Cases']
en = ElasticNet()
en.fit(X.values.reshape(-1, 1), y.values)
 preds = en.predict(X_pred.values.reshape(-1, 1))
tt.loc[(tt['Place'] == myplace)&(tt['Days_Since_Ten_Cases'] >= 0), 'ConfirmedCases_Log_Pred1'] = preds
tt.loc[(tt['Place'] == myplace), 'ConfirmedCases_Pred1'] = tt['ConfirmedCases_Log_Pred1'].apply(np.expm1)
 pop_myplace = tt.query('Place == @myplace')['Population_final'].values[0]
tt.loc[(tt['Place'] == myplace)&(tt['ConfirmedCases_Pred1'] >(0.1 * pop_myplace)) , 'ConfirmedCases_Pred1'] =(0.1 * pop_myplace)
 ax = tt.query('Place == @myplace' ).set_index('Date')[['ConfirmedCases','ConfirmedCases_Pred1']].plot(figsize=(15, 5), title=myplace, ax=axs[0])
 dat = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','Fatalities_Log']].dropna()
if len(dat)< 5:
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt.loc[(tt['Place'] == myplace)]['ConfirmedCases_Pred1'] * 0.001
elif tt.query('Place == @myplace')['Fatalities'].max() < 5:
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt.loc[(tt['Place'] == myplace)]['ConfirmedCases_Pred1'] * 0.001
else:
X = dat['Days_Since_Ten_Cases']
y = dat['Fatalities_Log']
y = y.cummax()
dat_all = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','Fatalities_Log']]
X_pred = dat_all['Days_Since_Ten_Cases']
en = ElasticNet()
en.fit(X.values.reshape(-1, 1), y.values)
 preds = en.predict(X_pred.values.reshape(-1, 1))
tt.loc[(tt['Place'] == myplace)&(tt['Days_Since_Ten_Cases'] >= 0), 'Fatalities_Log_Pred1'] = preds
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt['Fatalities_Log_Pred1'].apply(np.expm1)
 pop_myplace = tt.query('Place == @myplace')['Population_final'].values[0]
tt.loc[(tt['Place'] == myplace)&(tt['Fatalities_Pred1'] >(0.0005 * pop_myplace)) , 'Fatalities_Pred1'] =(0.0005 * pop_myplace)
 ax = tt.query('Place == @myplace' ).set_index('Date')[['Fatalities','Fatalities_Pred1']].plot(figsize=(15, 5), title=myplace, ax=axs[1])
 plt.show()
except:
print(f'============= FAILED FOR {myplace} =============')
 ax = tt.query('Place == @myplace' ).set_index('Date')[['ConfirmedCases','ConfirmedCases_Pred1']].plot(figsize=(15, 5), title=myplace, ax=axs[0])
 ax = tt.query('Place == @myplace' ).set_index('Date')[['Fatalities','Fatalities_Pred1']].plot(figsize=(15, 5), title=myplace, ax=axs[1] )
xgb_bayes_params = {'colsample_bytree': 0.3706219857878677,
'learning_rate': 0.16624226726409647,
'max_bin': 93400,
'max_depth': 134,
'min_child_samples': 22,
'min_child_weight': 4,
'n_estimators': 8028,
'num_leaves': 27,
'reg_alpha': 1.081049236893711e-05,
'reg_lambda': 1.043686239159047,
'scale_pos_weight': 0.19222548462579486,
'subsample': 0.6941640075502717,
'subsample_for_bin': 375140,
'subsample_freq': 7}
params=xgb_bayes_params<choose_model_class>
tt['ConfirmedCases_Pred1'] = tt[['ConfirmedCases','ConfirmedCases_Pred1']].max(axis=1)
 tt['Fatalities_Pred1'] = tt[['Fatalities','Fatalities_Pred1']].max(axis=1)
 tt['ConfirmedCases_Pred1'] = tt['ConfirmedCases_Pred1'].fillna(0)
 tt['Fatalities_Pred1'] = tt['Fatalities_Pred1'].fillna(0)
 tt.loc[~tt['ConfirmedCases'].isna() , 'ConfirmedCases_Pred1'] = tt.loc[~tt['ConfirmedCases'].isna() ]['ConfirmedCases']
tt.loc[~tt['Fatalities'].isna() , 'Fatalities_Pred1'] = tt.loc[~tt['Fatalities'].isna() ]['Fatalities']
tt['ConfirmedCases_Pred1'] = tt.groupby('Place')['ConfirmedCases_Pred1'].transform('cummax')
 tt['Fatalities_Pred1'] = tt.groupby('Place')['Fatalities_Pred1'].transform('cummax' )
counter = 0
xgb_model = xgb.XGBRegressor(
 metric='mae',
n_jobs=1,
verbose=0 
)
 dimensions = [Real(0.01, 1.0, 'log-uniform', name='learning_rate'),
Integer(2, 500, name='num_leaves'),
Integer(0, 500, name='max_depth'),
Integer(0, 200, name='min_child_samples'),
Integer(100, 100000, name='max_bin'),
Real(0.01, 1.0, 'uniform', name='subsample'),
Integer(0, 10, name='subsample_freq'),
Real(0.01, 1.0, 'uniform', name='colsample_bytree'),
Integer(0, 10, name='min_child_weight'),
Integer(100000, 500000, name='subsample_for_bin'),
Real(1e-9, 1000, 'log-uniform', name='reg_lambda'),
Real(1e-9, 1.0, 'log-uniform', name='reg_alpha'),
Real(1e-6, 500, 'log-uniform', name='scale_pos_weight'),
Integer(10, 10000, name='n_estimators')]
objective = make_objective(xgb_model,
scaled_train_X, train_y,
space=dimensions,
cv=k_fold,
scoring=mse_scoring )<prepare_x_and_y>
tt.groupby('Place')['Fatalities_Pred1'].max().sort_values()
PandasDataFrame = TypeVar('pandas.core.frame.DataFrame')
 def run_xgb(
 X_tr: PandasDataFrame,
X_val: PandasDataFrame,
y_tr: PandasDataFrame,
y_val: PandasDataFrame,
test_data: PandasDataFrame,
params: Dict 
):

 early_stop = 200
num_rounds = 10000
verbose_eval=1000
d_train = xgb.DMatrix(X_tr.values, label = y_tr)
 d_valid = xgb.DMatrix(X_val.values, label = y_val)
 d_test = xgb.DMatrix(test_data.values)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(params, dtrain=d_train,num_boost_round=num_rounds,evals=watchlist,
verbose_eval=verbose_eval,early_stopping_rounds=early_stop)
 imps=model.get_score(importance_type='gain')
 f_nums = model.feature_names
f_names = list(scaled_train_X)
 f_dict = dict(zip(f_nums,f_names))
keys=[]
for i in imps:
keys.append(f_dict.get(i))
X_tr_new = X_tr.loc[:,keys]
X_val_new=X_val.loc[:,keys]
test_data_new=test_data.loc[:,keys]
d_train = xgb.DMatrix(X_tr_new.values, label = y_tr)
 d_valid = xgb.DMatrix(X_val_new.values, label = y_val)
 d_test = xgb.DMatrix(test_data_new.values)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(params, dtrain=d_train,num_boost_round=num_rounds,evals=watchlist,
verbose_eval=verbose_eval,early_stopping_rounds=early_stop)
 val_pred = model.predict(d_valid)
 prediction = model.predict(d_test)
 return val_pred, prediction
def run_cv_model(
 train_data,
train_target,
test_data,
model_fn,
params,
eval_fn=None,
label='model',
feature_imp=False,
n_folds = 5 
):
oof_val = np.zeros(len(train_data))
predictions = np.zeros(len(test_data))
oof_predict = np.zeros(( n_folds, test_data.shape[0]))
scores = []
folds = KFold(n_splits=n_folds, shuffle=True, random_state=42)
 train_columns = train_data.columns.values
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_data, train_target.values)) :
strLog = "fold {} for {}".format(fold_, label)
 print(strLog)
 X_tr, X_val = train_data.iloc[trn_idx], train_data.iloc[val_idx]
y_tr, y_val = train_target.iloc[trn_idx], train_target.iloc[val_idx]
val_pred, prediction= model_fn(
 X_tr, X_val,
y_tr, y_val,
test_data,
params 
)
 score = mean_squared_error(y_val, val_pred)
 scores.append(score)
 oof_val[val_idx] = val_pred
if feature_imp == True:
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = train_columns
fold_importance_df["importance"] = feature_importances[:len(train_columns)]
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 oof_predict[fold_] = prediction
predictions += prediction/folds.n_splits
print('CV score: {0:.4f}, std: {1:.4f}.
 '.format(np.mean(score), np.std(score)))
 print('CV mean score: {0:.4f}, std: {1:.4f}.
 '.format(np.mean(scores), np.std(scores)))
 return oof_val, oof_predict, predictions<train_on_grid>
for myplace in ['Iran']:
fig, axs = plt.subplots(1, 2, figsize=(15, 3))
dat = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','ConfirmedCases']].dropna()
dat = dat.iloc[-10:]
X = dat['Days_Since_Ten_Cases']
y = dat['ConfirmedCases']
y = y.cummax()
dat_all = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','ConfirmedCases']]
X_pred = dat_all['Days_Since_Ten_Cases']
en = ElasticNet()
en.fit(X.values.reshape(-1, 1), y.values)
 preds = en.predict(X_pred.values.reshape(-1, 1))
tt.loc[(tt['Place'] == myplace)&(tt['Days_Since_Ten_Cases'] >= 0), 'ConfirmedCases_Pred1'] = preds
pop_myplace = tt.query('Place == @myplace')['Population_final'].values[0]
tt.loc[(tt['Place'] == myplace)&(tt['ConfirmedCases_Pred1'] >(0.1 * pop_myplace)) , 'ConfirmedCases_Pred1'] =(0.1 * pop_myplace)
 ax = tt.query('Place == @myplace' ).set_index('Date')[['ConfirmedCases','ConfirmedCases_Pred1']].plot(figsize=(15, 5), title=myplace, ax=axs[0])
 dat = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','Fatalities']].dropna()
dat = dat.iloc[-10:]
if len(dat)< 5:
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt.loc[(tt['Place'] == myplace)]['ConfirmedCases_Pred1'] * 0.001
elif tt.query('Place == @myplace')['Fatalities'].max() < 5:
tt.loc[(tt['Place'] == myplace), 'Fatalities_Pred1'] = tt.loc[(tt['Place'] == myplace)]['ConfirmedCases_Pred1'] * 0.001
else:
X = dat['Days_Since_Ten_Cases']
y = dat['Fatalities']
y = y.cummax()
dat_all = tt.query('Place == @myplace and Days_Since_Ten_Cases >= 0')[['Days_Since_Ten_Cases','Fatalities']]
X_pred = dat_all['Days_Since_Ten_Cases']
en = ElasticNet()
en.fit(X.values.reshape(-1, 1), y.values)
 preds = en.predict(X_pred.values.reshape(-1, 1))
tt.loc[(tt['Place'] == myplace)&(tt['Days_Since_Ten_Cases'] >= 0), 'Fatalities_Pred1'] = preds
pop_myplace = tt.query('Place == @myplace')['Population_final'].values[0]
tt.loc[(tt['Place'] == myplace)&(tt['Fatalities_Pred1'] >(0.0005 * pop_myplace)) , 'Fatalities_Pred1'] =(0.0005 * pop_myplace)
 ax = tt.query('Place == @myplace' ).set_index('Date')[['Fatalities','Fatalities_Pred1']].plot(figsize=(15, 5), title=myplace, ax=axs[1])
 plt.show()
fix_xgb_params = {
"boosting": "gblinear",
"metric": 'mae',
}
xgb_params = {**fix_xgb_params, **xgb_bayes_params}
oof_val, oof_predict, predictions = run_cv_model(
 train_data=scaled_train_X,
train_target=train_y,
test_data=scaled_test_X,
model_fn=run_xgb,
params=xgb_params,
eval_fn=None,
label="xgb",
feature_imp=False,
n_folds=10 
)
<save_to_csv>
tt['ConfirmedCases_Pred1'] = tt[['ConfirmedCases','ConfirmedCases_Pred1']].max(axis=1)
 tt['Fatalities_Pred1'] = tt[['Fatalities','Fatalities_Pred1']].max(axis=1)
 tt['ConfirmedCases_Pred1'] = tt['ConfirmedCases_Pred1'].fillna(0)
 tt['Fatalities_Pred1'] = tt['Fatalities_Pred1'].fillna(0)
 tt.loc[~tt['ConfirmedCases'].isna() , 'ConfirmedCases_Pred1'] = tt.loc[~tt['ConfirmedCases'].isna() ]['ConfirmedCases']
tt.loc[~tt['Fatalities'].isna() , 'Fatalities_Pred1'] = tt.loc[~tt['Fatalities'].isna() ]['Fatalities']
tt['ConfirmedCases_Pred1'] = tt.groupby('Place')['ConfirmedCases_Pred1'].transform('cummax')
 tt['Fatalities_Pred1'] = tt.groupby('Place')['Fatalities_Pred1'].transform('cummax' )
submission.time_to_failure = predictions
submission.to_csv('submission.csv',index=True )<set_options>
ss = pd.read_csv('.. /input/covid19-global-forecasting-week-2/submission.csv' )
warnings.filterwarnings("ignore")
 PandasDataFrame = TypeVar('pandas.core.frame.DataFrame')
<load_from_csv>
mysub = tt.dropna(subset=['ForecastId'])[['ForecastId','ConfirmedCases_Pred1','Fatalities_Pred1']]
mysub['ForecastId'] = mysub['ForecastId'].astype('int')
 mysub = mysub.rename(columns={'ConfirmedCases_Pred1':'ConfirmedCases',
'Fatalities_Pred1': 'Fatalities'})
 mysub.to_csv('submission.csv', index=False )