Kaggle competition: Bike Sharing Demand
data_dir = '/kaggle/input/covid19-global-forecasting-week-1'
pd.read_csv(f'{data_dir}/train.csv', parse_dates=['Date'] ).head()<load_from_csv>
df = pd.read_csv('.. /input/train.csv', parse_dates=[0] )
COLUMN_NAME_MAP = {
'Id': 'id',
'ForecastId': 'id',
'Province/State': 'province',
'Country/Region': 'country',
'Lat': 'lat',
'Long': 'long',
'Date': 'date',
'ConfirmedCases': 'cases',
'Fatalities': 'deaths'
}
def get_data(fn):

 df =(
 pd 
.read_csv(f'{data_dir}/{fn}.csv', parse_dates=['Date'])
.rename(columns=COLUMN_NAME_MAP)
.set_index('id')
.replace(pd.np.nan, '')
)
 df['day'] = df.date.dt.dayofyear
df['day'] -= 22
if 'deaths' in df:
df = df.astype({'deaths': int, 'cases': int})
 return df
train_df = get_data('train')
 train_df.head()<filter>
df = pd.read_csv('.. /input/train.csv', parse_dates=[0] )
train_df = train_df.loc[~bad_rows]<groupby>
test = pd.read_csv('.. /input/test.csv', parse_dates=[0] )
countries_train_df = train_df.groupby(['country', 'date', 'day'])['cases', 'deaths'].sum().reset_index()<sort_values>
test = pd.read_csv('.. /input/test.csv', parse_dates=[0] )
most_cases_df = countries_train_df.groupby('country')['cases', 'deaths'].max().sort_values('cases', ascending=False)
 most_cases_df.head(10 )<find_best_params>
df.rename(columns={'count': 'rentals'}, inplace=True )
test_df = get_data('test')
 x_predict = np.sort(test_df.day.unique())
 x_all = np.arange(x_predict.max() + 1)
 print(f'There are {len(x_predict)} to predict.')
 print(f'Overlap of {train_df.day.max() - test_df.day.min() } days.')
 test_df.head()<train_model>
df['rentals'] = np.log(df['rentals'] )
class ConstantModel:

 def __init__(self):
pass
def fit(self, X, y, sample_weight=None):
self.value = y[0]
def predict(self, X):
return np.array([self.value] * X.shape[0])
 def __repr__(self):
return f'ConstanModel(value={self.value})'
def run_models(grp_df, feature, target, x_to_predict, model, **model_kwargs):
region_data = grp_df[[feature, target]].to_numpy()
x = region_data[:, 0].reshape(-1, 1)
 y = region_data[:, 1]
n_target_classes = np.unique(y ).shape[0]
model = model(**model_kwargs)if n_target_classes > 1 else ConstantModel()
model.fit(x, y)
 result = {
feature: x_predict,
target: model.predict(x_to_predict.reshape(-1, 1)) ,
f'{target}_model': model
}
return pd.DataFrame(result ).set_index(feature )<groupby>
1000 - 1100, 10 - 11
region_columns = ['country', 'province']
death_predictions_df =(
 train_df 
.groupby(region_columns)
.apply(run_models, feature='day', target='deaths', x_to_predict=x_predict, model=model, **model_kwargs)
.reset_index() 
)
 cases_predictions_df =(
 train_df 
.groupby(region_columns)
.apply(run_models, feature='day', target='cases', x_to_predict=x_predict, model=model, **model_kwargs)
.reset_index()
)<merge>
import math
merge_columns = ['country', 'province', 'day']
result_df =(
 test_df 
.reset_index() 
.merge(death_predictions_df, on=merge_columns)
.merge(cases_predictions_df, on=merge_columns)
)
 result_df<merge>
math.log(1000)- math.log(1100), math.log(10)- math.log(11 )
evaluation_df = train_df.merge(
 result_df,
on=merge_columns,
how='inner',
suffixes=('', '_predicted')
)[merge_columns + ['cases', 'cases_predicted', 'deaths', 'deaths_predicted']]
for model_type in('cases', 'deaths',):
formula = f'log({model_type}_predicted + 1)- log({model_type} + 1)'
formula = f'({formula})**2'
evaluation_df = evaluation_df.eval(f'{model_type}_rmsle = {formula}')
 score_columns = ['cases_rmsle', 'deaths_rmsle'] 
(evaluation_df[score_columns].mean() ** 0.5 ).mean()<sort_values>
pd.concat([df, test] )
(
evaluation_df 
.groupby(region_columns)[['cases_rmsle', 'deaths_rmsle']] 
.sum() 
.sort_values('cases_rmsle', ascending=False)
 )<save_to_csv>
df = pd.concat([df, test] )
submission_df =(
 result_df 
.copy() 
.astype({'deaths': int, 'cases': int})
.rename(columns={v: k for k, v in COLUMN_NAME_MAP.items() })
.filter(['ForecastId', 'ConfirmedCases', 'Fatalities'])
)
 expected_submission_df = pd.read_csv(f'{data_dir}/submission.csv')
 for col in('ConfirmedCases', 'Fatalities',):
expected_submission_df[col] = submission_df[col]
pd.testing.assert_frame_equal(submission_df, expected_submission_df)
 submission_df.to_csv('submission.csv', index=False )<data_type_conversions>
df['year'] = df.datetime.dt.year
df['hour'] = df.datetime.dt.hour
df['dayofweek'] = df.datetime.dt.dayofweek
for model_type in('cases', 'deaths',):
train_df[f'daily_{model_type}'] = train_df.groupby(region_columns)[model_type].diff().fillna(0 ).astype(int)
 train_df.query('country == "United Kingdom" and province == "United Kingdom"' ).plot(x='day', y=f'daily_{model_type}' )<set_options>
test = df[df['rentals'].isnull() ]
df = df[~df['rentals'].isnull() ]
mpl.rcParams['figure.figsize'] =(8, 6)
 mpl.rcParams['axes.grid'] = False<categorify>
from sklearn.model_selection import train_test_split
def univariate_data(df, start_index, end_index, history_size, target_size):

 data = []
labels = []
dataset = df.to_numpy()
start_index = start_index + history_size
if end_index is None:
end_index = len(dataset)- target_size
if end_index <= start_index:
raise ValueError(f'End index {end_index} not greater than start index {start_index}.')
 for i in range(start_index, end_index):
indices = range(i-history_size, i)
 data.append(dataset[indices].reshape(history_size, 1))
labels.append(dataset[i+target_size])
 return data, labels
def multivariate_data(dataset, target, start_index, end_index, history_size, target_size, step=1):
data = []
labels = []
dataset = dataset.to_numpy()
target = target.to_numpy()
start_index = start_index + history_size
if end_index is None:
end_index = len(dataset)- target_size
if end_index <= start_index:
raise ValueError(f'End index {end_index} not greater than start index {start_index}.')
 for i in range(start_index, end_index):
indices = range(i-history_size, i, step)
 data.append(dataset[indices].reshape(history_size, -1))
if step > 1:
labels.append(target[i:i+target_size])
 else:
labels.append(target[i+target_size])
 return data, labels<prepare_x_and_y>
from sklearn.model_selection import train_test_split
history = 5
forecast_jump = 3
features = 'day'
scale = True
train_sample_size = test_df['day'].min()
scaled_df = train_df.copy()
if scale:
scaled_df[features] =(scaled_df[features] - scaled_df[features].mean())/ scaled_df[features].std()
xs = []
ys = []
regions = []
for grp_name, grp_df in scaled_df.groupby(['country', 'province']):
grp_x, grp_y = univariate_data(grp_df[features], 0, train_sample_size, history, forecast_jump)
 xs += grp_x
ys += grp_y
regions += [grp_name] * len(grp_y)
 break
x_train_uni = np.array(xs)
 y_train_uni = np.array(ys)
 regions_train = np.array(regions )<predict_on_test>
train, valid = train_test_split(df, random_state=42 )
print('Single window of past history')
 print(x_train_uni[0])
 print('
Target temperature to predict')
 print(y_train_uni[0] )<prepare_x_and_y>
train, valid = train_test_split(df, random_state=42 )
scaled_df = train_df.copy()
if scale:
scaled_df[features] =(scaled_df[features] - scaled_df[features].mean())/ scaled_df[features].std()
xs = []
ys = []
regions = []
for grp_name, grp_df in scaled_df.groupby(['country', 'province']):
grp_x, grp_y = multivariate_data(
 grp_df[features], grp_df[features], train_sample_size, None, history, forecast_jump 
)
 xs += grp_x
ys += grp_y
regions += [grp_name] * len(grp_y)
 x_val_uni = np.array(xs)
 y_val_uni = np.array(ys)
 regions_val = np.array(regions )<predict_on_test>
train.count()
i = 0
print('Single window of past history')
 print(x_val_uni[i])
 print('
Target temperature to predict')
 print(y_val_uni[i] )<split>
valid.count()
BATCH_SIZE = 256
BUFFER_SIZE = 10000
train_univariate = tf.data.Dataset.from_tensor_slices(( x_train_uni, y_train_uni))
train_univariate = train_univariate.cache().shuffle(BUFFER_SIZE ).batch(BATCH_SIZE ).repeat()
val_univariate = tf.data.Dataset.from_tensor_slices(( x_val_uni, y_val_uni))
val_univariate = val_univariate.batch(BATCH_SIZE ).repeat()<choose_model_class>
removed_cols = ['rentals', 'casual', 'registered', 'datetime']
simple_lstm_model = tf.keras.models.Sequential([
tf.keras.layers.LSTM(8, input_shape=x_train_uni.shape[-2:]),
tf.keras.layers.Dense(1)
 ])
 simple_lstm_model.compile(optimizer='adam', loss='mae' )<train_model>
feats = [c for c in df.columns if c not in removed_cols]
EVALUATION_INTERVAL = 200
EPOCHS = 10
single_step_history = simple_lstm_model.fit(train_univariate, epochs=EPOCHS,
steps_per_epoch=EVALUATION_INTERVAL,
validation_data=val_univariate, validation_steps=50);<import_modules>
from sklearn.tree import DecisionTreeRegressor
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit
from statsmodels.tsa.statespace.sarimax import SARIMAX
from statsmodels.tsa.arima_model import ARIMA
from random import random<drop_column>
dt = DecisionTreeRegressor(random_state=42, max_depth=2 )
df=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/train.csv",usecols=['Lat','Province/State','Country/Region','Date','ConfirmedCases','Fatalities'])
 df.fillna(' ',inplace=True)
 df['Lat']=df['Province/State']+df['Country/Region']
df.drop('Province/State',axis=1,inplace=True)
 df.drop('Country/Region',axis=1,inplace=True)
 countries_list=df.Lat.unique()
df1=[]
for i in countries_list:
df1.append(df[df['Lat']==i])
 print("we have "+ str(len(df1)) +" regions in our dataset" )<train_model>
dt.fit(train[feats],train['rentals'] )
submit_confirmed=[]
submit_fatal=[]
for i in df1:
data = i.ConfirmedCases.astype('int32' ).tolist()
try:
model = SARIMAX(data, order=(2,1,0), seasonal_order=(1,1,0,12))
model_fit = model.fit(disp=False)
 predicted = model_fit.predict(len(data), len(data)+34)
 new=np.concatenate(( np.array(data),np.array([int(num)for num in predicted])) ,axis=0)
 submit_confirmed.extend(list(new[-43:]))
except:
submit_confirmed.extend(list(data[-10:-1]))
for j in range(34):
submit_confirmed.append(data[-1]*2)
 data = i.Fatalities.astype('int32' ).tolist()
try:
model = SARIMAX(data, order=(2,1,0), seasonal_order=(1,1,0,12))
model_fit = model.fit(disp=False)
 predicted = model_fit.predict(len(data), len(data)+34)
 new=np.concatenate(( np.array(data),np.array([int(num)for num in predicted])) ,axis=0)
 submit_fatal.extend(list(new[-43:]))
except:
submit_fatal.extend(list(data[-10:-1]))
for j in range(34):
submit_fatal.append(data[-1]*2)
 df_submit=pd.concat([pd.Series(np.arange(1,1+len(submit_confirmed))),pd.Series(submit_confirmed),pd.Series(submit_fatal)],axis=1 )<save_to_csv>
from fastai.structured import draw_tree
df_submit=df_submit.fillna(method='pad' ).astype(int)
 df_submit.to_csv('submission.csv',header=['ForecastId','ConfirmedCases','Fatalities'],index=False )<set_options>
from sklearn.metrics import mean_squared_error
import scipy.optimize as opt
import matplotlib.pyplot as plt
from datetime import datetime, timedelta
from tqdm import tqdm_notebook
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_squared_log_error
import warnings; warnings.filterwarnings('ignore' )<load_from_csv>
from sklearn.metrics import mean_squared_error
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/train.csv')
 train['Date'] = pd.to_datetime(train['Date'])
 last_train_date = np.datetime64('2020-03-11')
 eval_set = train[train['Date'] > last_train_date]
overlap_days = eval_set['Date'].unique() [-1] - last_train_date
overlap_days = int(overlap_days.astype('timedelta64[D]')/ np.timedelta64(1, 'D'))
train<load_from_csv>
train['preds'] = train['rentals'].mean()
test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/test.csv')
 test['Date'] = pd.to_datetime(test['Date'])
 test<load_from_csv>
mean_squared_error(train['rentals'], train['preds'] )
sub = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/submission.csv')
 sub = sub.set_index('ForecastId', drop=True)
 sub<compute_test_metric>
del train['preds']
def model(parameters, time):
y_pred = parameters[0] *(1 - np.exp(-parameters[1] *(time - parameters[3])))** parameters[2]
return np.nan_to_num(y_pred ).clip(0, np.inf)
 def residual(parameters, time, data):
y_pred = model(parameters, time)
 return mean_squared_error(data, y_pred )<train_model>
from sklearn.ensemble import RandomForestRegressor
def fitModel(time, data, guess):
params = opt.minimize(residual, guess, args=(time, data), bounds=opt.Bounds(0, np.inf, True), method='Nelder-Mead', tol=1e-7)
 return params.x<train_model>
from sklearn.ensemble import RandomForestRegressor
model_params = trainModels(train )<predict_on_test>
rf = RandomForestRegressor(random_state=42, n_jobs=-1 )
model_predictions = forecast(test, model_params, sub )<merge>
rf.fit(train[feats], train['rentals'] )
eval_with_id = pd.merge(eval_set, test, on=['Date', 'Country/Region', 'Province/State'])
 sub = sub.replace([np.inf, -np.inf], np.nan)
 sub = sub.fillna(0)
 merged_eval = pd.merge(eval_with_id, sub, left_on='ForecastId', right_index=True)
 merged_eval<compute_test_metric>
train_preds = rf.predict(train[feats] )
score_confirmed = np.sqrt(mean_squared_log_error(merged_eval['ConfirmedCases_x'].values, merged_eval['ConfirmedCases_y']))
score_fatality = np.sqrt(mean_squared_log_error(merged_eval['Fatalities_x'].values, merged_eval['Fatalities_y']))
print(f'Confirmed Cases Score: {score_confirmed}
Fatality Score: {score_fatality}
Average Score: {np.mean([score_confirmed, score_fatality])}' )<save_to_csv>
train_preds = rf.predict(train[feats] )
sub.to_csv('submission.csv' )<load_from_csv>
mean_squared_error(train['rentals'], train_preds)**(1/2 )
train_df = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/train.csv")
 test_df = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/test.csv" )<count_unique_values>
valid_preds = rf.predict(valid[feats] )
country_list = train_df['Country/Region'].unique()
number_list = list(range(len(country_list)))
 country_num_map = dict(zip(country_list, number_list))<categorify>
mean_squared_error(valid['rentals'], valid_preds)**(1/2 )
train_df['Country_id'] = train_df['Country/Region'].map(country_num_map)
 test_df['Country_id'] = test_df['Country/Region'].map(country_num_map )<import_modules>
test['count'] = np.exp(rf.predict(test[feats]))
<feature_engineering><EOS> 
test[['datetime','count']].to_csv('rf.csv', index=False )