Kaggle competition: COVID19 Global Forecasting (Week 1)
gc.collect()<load_from_csv>
%matplotlib inline
def sigmoid_sqrt_func(x, a, b, c, d, e):
return c + d /(1.0 + np.exp(-a*x+b)) + e*x**0.5
def sigmoid_linear_func(x, a, b, c, d, e):
return c + d /(1.0 + np.exp(-a*x+b)) + e*0.1*x
def sigmoid_quad_func(x, a, b, c, d, e, f):
return c + d /(1.0 + np.exp(-a*x+b)) + e*0.1*x + f*0.001*x*x
def sigmoid_func(x, a, b, c, d):
return c + d /(1.0 + np.exp(-a*x+b))
def exp_func(x, a, b, c, d):
return c + d * np.exp(a*x+b)
 def func_fitting(y, func=sigmoid_func, x_scale=50.0, y_scale=10000.0, start_pred=8, AN=0, MAXN=60, PN=15, b=5):
x = range(len(y))
x_real = np.array(x)/x_scale
y_real = np.array(y)/y_scale
x_train = x_real
y_train = y_real
def next_day_pred(AN, BN):
x_train = x_real[AN:BN]
y_train = y_real[AN:BN]
popt, pcov = curve_fit(func, x_train, y_train,
method='trf',
maxfev=20000,
p0=(1, 0, 0, 1),
bounds=[(-b, -np.inf, -np.inf, -b),(b, np.inf, np.inf, b)], 
)
 x_pred = np.array(range(MAXN)) /x_scale
y_pred = func(x_pred, *popt)
 return x_pred, y_pred
NP = start_pred
y_pred = [np.nan]*NP
y_pred_list = []
for BN in range(NP, len(y_real)) :
x_pred, y_pred_ = next_day_pred(BN-PN, BN)
 y_pred.append(y_pred_[BN])
 y_pred_list.append(y_pred_)
 for BN in range(len(y_real), len(y_pred_)) :
y_pred.append(y_pred_[BN])
 y_pred = np.array(y_pred)
 y_pred_list = np.array(y_pred_list)
 y_pred_std = np.std(y_pred_list[-2:], axis=0)
 return x_real*x_scale, y_real*y_scale, x_train*x_scale, y_train*y_scale, \
x_pred*x_scale, y_pred*y_scale, y_pred_std*y_scale
def draw_figure(start_date, title, x_real, y_real, x_train, y_train, x_pred, y_pred, y_pred_std):
def to_date(idx):
idx = np.round(idx)
 return datetime.datetime.strptime(start_date, '%m/%d/%Y' ).date() + datetime.timedelta(days=idx)
 fig, ax1 = plt.subplots(figsize=[14, 7])
 plot1 = ax1.plot(list(map(to_date, x_real)) , y_real, 'gs',label='original')
 plot2 = ax1.plot(list(map(to_date, x_pred)) , y_pred, 'r',label='predict')
 plot3 = ax1.fill_between(list(map(to_date, x_pred)) ,
np.maximum(0,(y_pred-y_pred_std)) , 
(y_pred+y_pred_std),
alpha=0.2, edgecolor='
plot0 = ax1.plot(list(map(to_date, x_train)) , y_train, 'y.',label='history')
 ax2=ax1.twinx()
ax2.plot(list(map(to_date, x_real)) [1:],(y_real[1:]-y_real[:-1]), '-s',label='original add')
 ax2.plot(list(map(to_date, x_pred)) [1:],(y_pred[1:]-y_pred[:-1]), '-',label='pred add')
 plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%m-%d'))
plt.gca().xaxis.set_major_locator(mdates.DayLocator())
 plt.gcf().autofmt_xdate()
plt.xlabel('x')
 plt.ylabel('y')
 fig.legend(loc=2)
 plt.title(title)
 plt.savefig('{}.pdf'.format(title))
plt.show()
date = list(map(to_date, x_pred))
pred = y_pred
real = y_real
for i in range(len(pred)) :
if i < len(real):
print('{}\t{:.0f}\t{:.0f}\t{:.3f}'.format(date[i], real[i], pred[i], np.abs(pred[i]-real[i])/real[i]*100))
else:
print('{}\t-\t{:.0f}'.format(date[i], pred[i]))
return pred
x_test = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 rows = x_test.shape[0]
preds_start = np.zeros(( rows,max_len))
preds_end = np.zeros(( rows,max_len))<categorify>
train_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/train.csv')
 test_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/test.csv')
 pred_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/submission.csv')
 train_data = train_data.fillna(value='NULL')
 test_data = test_data.fillna(value='NULL' )
rows = x_test.shape[0]
test_ids = np.zeros(( rows,max_len), dtype = 'int32')
 test_mask = np.zeros(( rows,max_len), dtype = 'int32')
 test_type_ids = np.zeros(( rows,max_len), dtype = 'int32')
 test_start_idx = np.zeros(( rows,max_len), dtype = 'int32')
 test_end_idx = np.zeros(( rows,max_len), dtype = 'int32')
 for i in range(x_test.shape[0]):
encoding = text_process(x_test.loc[i,'text'], x_test.loc[i,'sentiment'], tokenizer, max_len)
 test_ids[i] = encoding['ids']
test_type_ids[i] = encoding['token_type_ids']
test_mask[i] = encoding['mask'] 
<predict_on_test>
train_date_list = train_data.iloc[:, 5].unique()
print(len(train_date_list))
print(train_date_list)
 test_date_list = test_data.iloc[:, 5].unique()
print(len(test_date_list))
print(test_date_list)
 len(train_data.groupby(['Province/State', 'Country/Region']))
len(test_data.groupby(['Province/State', 'Country/Region']))
for i in range(n_splits):
model.load_weights(f'.. /input/roberta-folds/roberta-{i}.h5')
 preds = model.predict([test_ids, test_mask, test_type_ids], verbose = True)
 preds_start += preds[0]/n_splits
preds_end += preds[1] /n_splits<load_from_csv>
start_date = '01/22/2020'
test_date_list = test_data.iloc[:, 5].unique()
test_data_filled = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/test.csv')
 test_data_filled = test_data_filled.fillna(value='NULL')
 test_data_filled['ConfirmedCases'] = pred_data['ConfirmedCases']
test_data_filled['Fatalities'] = pred_data['Fatalities']
for idx,(k, v)in enumerate(train_data.groupby(['Province/State', 'Country/Region'])) :
print(idx, k, end=' ')
 b_cc, b_f = 5, 3
if k[1] == 'China':
b_cc = 1
b_f = 1
if k[0] == 'Hong Kong':
b_cc = 3
elif k[1] == 'Italy':
b_cc = 10
b_f = 3
elif k[1] == 'US':
b_cc = 5
b_f = 3
elif k[1] == 'Spain':
b_cc = 4
b_f = 3
hist_num = v.loc[:,'ConfirmedCases'].tolist()
ret = func_fitting(hist_num, y_scale=max(1000, np.max(hist_num)) , b=b_cc,
start_pred=60, PN=10, MAXN=len(hist_num)+30)
 ret = list(ret)
 real_cc = np.round(np.array(ret[1]))
pred_cc = np.round(np.array(ret[5]))
for i in range(len(real_cc)) :
pred_cc[i] = real_cc[i]
pred_cc = pred_cc[50:]
print(pred_cc)
 hist_num = v.loc[:,'Fatalities'].tolist()
ret = func_fitting(hist_num, y_scale=max(1000, np.max(hist_num)) , b=b_f,
start_pred=60, PN=10, MAXN=len(hist_num)+30)
 ret = list(ret)
 real_f = np.round(np.array(ret[1]))
pred_f = np.round(np.array(ret[5]))
for i in range(len(real_f)) :
pred_f[i] = real_f[i]
pred_f = pred_f[50:]
print(pred_f)
 for i in range(14, len(pred_cc)) :
if pred_f[i] < 20 and pred_cc[i] > 200 and k[1] != 'China':
pred_f[i] = pred_cc[i] * 0.01 * np.log10(pred_cc[i])
 print(pred_f)
 print(pred_cc[-1], pred_f[-1])
 for i in range(len(pred_cc)) :
index =(test_data_filled['Province/State'] == k[0])& \ 
(test_data_filled['Country/Region'] == k[1])& \ 
(test_data_filled['Date'] == test_date_list[i])
 test_data_filled.loc[index, 'ConfirmedCases'] = pred_cc[i]
test_data_filled.loc[index, 'Fatalities'] = pred_f[i]
submission = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv' )<categorify>
submission = test_data_filled.loc[:,['ForecastId', 'ConfirmedCases', 'Fatalities']]
for k in range(x_test.shape[0]):
if x_test.loc[k,'sentiment']!='neutral':
encoding = text_process(x_test.loc[k,'text'], x_test.loc[k,'sentiment'], tokenizer, max_len)
 offsets = encoding['offsets']
targets_start = np.argmax(preds_start[k,])
 targets_end = np.argmax(preds_end[k,])
 if targets_start<=targets_end:
pred = x_test.loc[k,'text'][offsets[targets_start][0]:offsets[targets_end][-1]]
submission.loc[k,'selected_text'] = pred
else:
submission.loc[k,'selected_text'] = x_test.loc[k,'text']
else:
submission.loc[k,'selected_text'] = x_test.loc[k,'text'] 
<save_to_csv>
submission.to_csv("submission.csv", index=False)
 submission.head(500 )
submission.to_csv('submission.csv', index = False )<set_options>
np.log10(80000 )