Kaggle competition: COVID19 Global Forecasting (Week 1)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-1)<predict_on_test>
path = '.. /input/covid19-global-forecasting-week-1/'
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sub = pd.read_csv(path + 'submission.csv')
 train['Date'] = train['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 test['Date'] = test['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 path_ext = '.. /input/novel-corona-virus-2019-dataset/'
ext_rec = pd.read_csv(path_ext + 'time_series_covid_19_recovered.csv' ).\
melt(id_vars=["Province/State", "Country/Region", "Lat", "Long"],
var_name="Date",
value_name="Recoveries")
 ext_rec['Date'] = ext_rec['Date'].apply(lambda x:(datetime.datetime.strptime(x+"20", '%m/%d/%Y')))
 train = train.merge(ext_rec[['Province/State', 'Country/Region', 'Date', 'Recoveries']], how='left',
left_on=['Province/State', 'Country/Region', 'Date'],
right_on=['Province/State', 'Country/Region', 'Date'])
 train['days'] =(train['Date'].dt.date - train['Date'].dt.date.min() ).dt.days
test['days'] =(test['Date'].dt.date - train['Date'].dt.date.min() ).dt.days
train.loc[train['Province/State'].isnull() , 'Province/State'] = 'N/A'
test.loc[test['Province/State'].isnull() , 'Province/State'] = 'N/A'
train['Area'] = train['Country/Region'] + '_' + train['Province/State']
test['Area'] = test['Country/Region'] + '_' + test['Province/State']
print(train['Date'].max())
 print(train['days'].max())
 N_AREAS = train['Area'].nunique()
AREAS = np.sort(train['Area'].unique())
 TRAIN_N = 50 + 13
print(train[train['days'] < TRAIN_N]['Date'].max())
 train.head()
predictions = predict_on_video_set(test_videos, num_workers=4 )<save_to_csv>
train_p_c = train.pivot(index='Area', columns='days', values='ConfirmedCases' ).sort_index()
train_p_f = train.pivot(index='Area', columns='days', values='Fatalities' ).sort_index()
train_p_c = np.maximum.accumulate(train_p_c, axis=1)
 train_p_f = np.maximum.accumulate(train_p_f, axis=1)
 f_rate =(train_p_f / train_p_c ).fillna(0)
 X_c = np.log(1+train_p_c.values)
 X_f = train_p_f.values
submission_df = pd.DataFrame({"filename": test_videos, "label": predictions})
 submission_df.to_csv("submission.csv", index=False )<set_options>
def eval1(y, p):
val_len = y.shape[1] - TRAIN_N
return np.sqrt(mean_squared_error(y[:, TRAIN_N:TRAIN_N+val_len].flatten() , p[:, TRAIN_N:TRAIN_N+val_len].flatten()))
def run_c(params, X, test_size=50):
gr_base = []
for i in range(X_c.shape[0]):
temp = X[i,:]
threshold = np.log(1+params['min cases for growth rate'])
 num_days = params['last N days']
if(temp > threshold ).sum() > num_days:
gr_base.append(np.clip(np.diff(temp[temp > threshold])[-num_days:].mean() , 0, params['growth rate max']))
else:
gr_base.append(params['growth rate default'])
 gr_base = np.array(gr_base)
 preds = X.copy()
for i in range(test_size):
delta = np.clip(preds[:, -1], np.log(2), None)+ gr_base *(1 + params['growth rate factor'])**i
preds = np.hstack(( preds, delta.reshape(-1,1)))
 return preds
params = {
"min cases for growth rate": 30,
"last N days": 5,
"growth rate default": 0.20,
"growth rate max": 0.3,
"growth rate factor": -0.03,
}
preds_c = run_c(params, np.log(1+train_p_c.values)[:,:TRAIN_N])
warnings.filterwarnings('ignore')
 pd.set_option("max_columns", 500)
 %matplotlib inline<load_from_csv>
def run_f(params, X_c, X_f, X_f_r, test_size=50):
X_f_r = np.array(np.ma.mean(np.ma.masked_outside(X_f_r, 0.06, 0.4)[:,:], axis=1))
X_f_r = np.clip(X_f_r, params['fatality_rate_lower'], params['fatality_rate_upper'])
 X_c = np.clip(np.exp(X_c)-1, 0, None)
 preds = X_f.copy()
train_size = X_f.shape[1] - 1
for i in range(test_size):
delta = X_c[:, train_size+i-params['length']+1] - X_c[:, train_size+i-params['length']]
delta = params['absolute growth'] + delta*X_f_r
preds = np.hstack(( preds, preds[:, -1].reshape(-1,1)+ delta.reshape(-1,1)))
 return preds
params = {
"length": 8,
"absolute growth": 0.02,
"fatality_rate_lower": 0.035,
"fatality_rate_upper": 0.40,
}
preds_f = run_f(params, preds_c, X_f, f_rate.values[:,:TRAIN_N])
 preds_f = np.log(1+preds_f)
train = pd.read_csv(".. /input/ashrae-energy-prediction/train.csv", parse_dates=["timestamp"])
 test = pd.read_csv(".. /input/ashrae-energy-prediction/test.csv", parse_dates=["timestamp"])
 building = pd.read_csv('.. /input/ashrae-energy-prediction/building_metadata.csv')
 weather_train = pd.read_csv('.. /input/ashrae-energy-prediction/weather_train.csv', parse_dates=["timestamp"])
 weather_test = pd.read_csv(".. /input/ashrae-energy-prediction/weather_test.csv", parse_dates=["timestamp"] )<merge>
if False:
val_len = train_p_c.values.shape[1] - TRAIN_N
for i in range(val_len):
d = i + TRAIN_N
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c.values[:, d]), preds_c[:, d]))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f.values[:, d]), preds_f[:, d]))
print(f"{d}: {(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]")
 print()
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_c[:, TRAIN_N:TRAIN_N+val_len].flatten()))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_f[:, TRAIN_N:TRAIN_N+val_len].flatten()))
print(f"{(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]" )
train_plot = train.merge(building, on='building_id', how='left')
 site = 0
meter_type = 1
primary_use = 'Education'
r = int(np.ceil(len(train_plot[(train_plot['site_id'] == site)&(train_plot['primary_use'] == primary_use)&(train_plot['meter'] == meter_type)]['building_id'].value_counts(dropna=False ).index.to_list())/2))
fig, axes = plt.subplots(r,2,figsize=(14, 36), dpi=100)
 for i, building_id in enumerate(train_plot[(train_plot['site_id'] == site)&(train_plot['primary_use'] == primary_use)&(train_plot['meter'] == meter_type)]['building_id'].value_counts(dropna=False ).index.to_list()):
train_plot[(train_plot['site_id'] == site)&(train_plot['primary_use'] == primary_use)&(train_plot['meter'] == meter_type)& 
(train_plot['building_id'] == building_id)][['timestamp', 'meter_reading']].set_index('timestamp' ).resample('H' ).mean() ['meter_reading'].plot(ax=axes[i%r][i//r],
alpha=0.8, label='By hour', color='tab:blue' ).set_ylabel('Mean meter reading', fontsize=13);
train_plot[(train_plot['site_id'] == site)&(train_plot['primary_use'] == primary_use)&(train_plot['meter'] == meter_type)& 
(train_plot['building_id'] == building_id)][['timestamp', 'meter_reading']].set_index('timestamp' ).resample('D' ).mean() ['meter_reading'].plot(ax=axes[i%r][i//r],
alpha=1, label='By day', color='tab:orange' ).set_xlabel('');
axes[i%r][i//r].legend() ;
axes[i%r][i//r].set_title('building_id: ' + str(building_id), fontsize=13);
plt.subplots_adjust(hspace=0.45)
 del train_plot,fig,axes,r
gc.collect() ;<data_type_conversions>
temp = pd.DataFrame(np.clip(np.exp(preds_c)- 1, 0, None))
temp['Area'] = AREAS
temp = temp.melt(id_vars='Area', var_name='days', value_name="ConfirmedCases")
 test = test.merge(temp, how='left', left_on=['Area', 'days'], right_on=['Area', 'days'])
 temp = pd.DataFrame(np.clip(np.exp(preds_f)- 1, 0, None))
temp['Area'] = AREAS
temp = temp.melt(id_vars='Area', var_name='days', value_name="Fatalities")
 test = test.merge(temp, how='left', left_on=['Area', 'days'], right_on=['Area', 'days'])
 test.head()
def compress_dataframe(df):

 result = df.copy()
for col in result.columns:
col_data = result[col]
dn = col_data.dtype.name
if dn == "object":
result[col] = pd.to_numeric(col_data.astype("category" ).cat.codes, downcast="integer")
 elif dn == "bool":
result[col] = col_data.astype("int8")
 elif dn.startswith("int")or(col_data.round() == col_data ).all() :
result[col] = pd.to_numeric(col_data, downcast="integer")
 else:
result[col] = pd.to_numeric(col_data, downcast='float')
 return result<data_type_conversions>
test.to_csv("submission.csv", index=False, columns=["ForecastId", "ConfirmedCases", "Fatalities"] )
<merge><EOS> 
for i, rec in test.groupby('Area' ).last().sort_values("ConfirmedCases", ascending=False ).iterrows() :
print(f"{rec['ConfirmedCases']:10.1f} {rec['Fatalities']:10.1f} {rec['Country/Region']}, {rec['Province/State']}")