Kaggle competition: IESB - 2019
print('Importing training and test data')
 train_df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/train.csv')
 test_df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv')
 train_df['Province_State'] = train_df['Province_State'].fillna('')
 train_df['Region'] = train_df['Country_Region'] + train_df['Province_State']
test_df['Province_State'] = test_df['Province_State'].fillna('')
 test_df['Region'] = test_df['Country_Region'] + test_df['Province_State']
regions = train_df.Region.unique()
train_min_date = train_df[train_df['Region']=='Sweden']['Date'].min()
test_min_date = test_df[test_df['Region']=='Sweden']['Date'].min()
dt_train_min = datetime.strptime(train_min_date, '%Y-%m-%d')
 dt_test_min = datetime.strptime(test_min_date, '%Y-%m-%d')
 test_start_day = dt_test_min.timetuple().tm_yday - dt_train_min.timetuple().tm_yday
print(test_start_day)
 train_data = {}
test_data = {}
for region in regions:
train_data[region] = train_df[train_df['Region']==region]
train_data[region]['DayNo'] = np.arange(len(train_df[train_df['Region']==region]['Date']))
test_data[region] = test_df[test_df['Region']==region]
test_data[region]['DayNo'] = np.arange(test_start_day,test_start_day+len(test_df[test_df['Region']==region]['Date'])) 
<define_variables>
test = pd.read_csv('.. /input/test.csv')
 train = df = pd.read_csv('.. /input/train.csv' )
train_max_date = train_df[train_df['Region']=='Sweden']['Date'].max()
dt_train_max = datetime.strptime(train_max_date, '%Y-%m-%d')
 train_max_day = dt_train_max.timetuple().tm_yday - dt_train_min.timetuple().tm_yday 
<compute_test_metric>
df = pd.concat([train, test] )
def sigmoid(x, a, b, c):
return a*np.exp(c*(x-b)) /(np.exp(c*(x-b)) +1 )<train_on_grid>
numero = [c for c in df.columns if c not in texto]
numero = [c for c in numero if c not in ['nota_mat', 'codigo_mun', 'Unnamed: 0']]
failed_confirmed = []
failed_fatalities = []
confirmed_popt = {}
fatalities_popt = {}
for region in regions:
x_data = train_data[region]['DayNo']
y_ConfirmedCases_data = train_data[region]['ConfirmedCases']
y_Fatalities_data = train_data[region]['Fatalities']
try:
popt, pcov = curve_fit(sigmoid, x_data, y_ConfirmedCases_data)
 confirmed_popt[region] = popt
except:
failed_confirmed.append(region)
 try:
popt, pcov = curve_fit(sigmoid, x_data, y_Fatalities_data)
 fatalities_popt[region] = popt
except:
failed_fatalities.append(region)
 print("Failed confirmed: " + str(len(failed_confirmed)))
 print("Failed fatalities: " + str(len(failed_fatalities)))
 print("Total: " + str(len(regions)) )<train_model>
def outliers_iqr(ys):
quartile_1, quartile_3 = np.percentile(ys, [25, 75])
 iqr = quartile_3 - quartile_1
lower_bound = quartile_1 -(iqr * 1.5)
 upper_bound = quartile_3 +(iqr * 1.5)
 return np.where(( ys > upper_bound)|(ys < lower_bound))
confirmed_coeffs = [x for x in confirmed_popt.values() ]
mean_confirmed_coeffs = np.mean(confirmed_coeffs, axis=0)
 print(mean_confirmed_coeffs)
 fatalities_coeffs = [x for x in fatalities_popt.values() ]
mean_fatalities_coeffs = np.mean(fatalities_coeffs, axis=0)
 print(mean_fatalities_coeffs)
 for region in failed_confirmed:
x_data = train_data[region]['DayNo']
y_ConfirmedCases_data = train_data[region]['ConfirmedCases']
try:
popt, pcov = curve_fit(sigmoid, x_data, y_ConfirmedCases_data, maxfev=1000, ftol=1e-5)
 confirmed_popt[region] = popt
except:
start = 0
for data in y_ConfirmedCases_data:
if data > 0:
break
start = start + 1
popt = mean_confirmed_coeffs
popt[1] = start
confirmed_popt[region] = popt
print("Failed for C " + region + " : " + str(popt))
for region in failed_fatalities:
x_data = train_data[region]['DayNo']
y_Fatalities_data = train_data[region]['Fatalities']
try:
popt, pcov = curve_fit(sigmoid, x_data, y_Fatalities_data, maxfev=1000, ftol=1e-5)
 fatalities_popt[region] = popt
except:
start = 0
for data in y_Fatalities_data:
if data > 0:
break
start = start + 1
popt = mean_fatalities_coeffs
popt[1] = start
fatalities_popt[region] = popt
print("Failed F for " + region + " : " + str(popt))<prepare_x_and_y>
for a in numero:
quartile_1, quartile_3 = np.percentile(df[a], [25, 75])
 iqr = quartile_3 - quartile_1
lower_bound = quartile_1 -(iqr * 1.5)
 upper_bound = quartile_3 +(iqr * 1.5)
 df[a][df[a] < lower_bound] = None
df[a][df[a] > upper_bound] = None
sub = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/submission.csv')
 test_regions = test_df.Region.unique()
total_count = 0
for region in test_regions:
forecastIds = test_data[region]['ForecastId']
x_test_data = test_data[region]['DayNo']
y_conf_test_data = np.nan_to_num(sigmoid(x_test_data, *confirmed_popt[region])).astype(np.int)
 x_test_data = test_data[region]['DayNo']
y_fatal_test_data = np.nan_to_num(sigmoid(x_test_data, *fatalities_popt[region])).astype(np.int)
 idx = 0
x_test_data = x_test_data.tolist()
for id in forecastIds:
day_no = x_test_data[idx]
row_index = sub.index[sub['ForecastId'] == id]
if day_no > train_max_day:
sub.set_value(row_index, 'ConfirmedCases', y_conf_test_data[idx])
 sub.set_value(row_index, 'Fatalities', y_fatal_test_data[idx])
 else:
sub.set_value(row_index, 'ConfirmedCases', int(train_data[region][train_data[region]['DayNo']==day_no]['ConfirmedCases'].tolist() [0]))
sub.set_value(row_index, 'Fatalities', int(train_data[region][train_data[region]['DayNo']==day_no]['Fatalities'].tolist() [0]))
idx = idx + 1
sub.to_csv('/kaggle/working/submission.csv', index=False )<load_from_csv>
texto = [c for c in texto if c not in ['codigo_mun']]
df_train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/train.csv')
 df_test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/test.csv')
 df_sub = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/test.csv' )<data_type_conversions>
for c in texto:
df[c]=df[c].astype('category' ).cat.codes
df_train['Date'] = pd.to_datetime(df_train['Date'], infer_datetime_format=True)
 df_test['Date'] = pd.to_datetime(df_test['Date'], infer_datetime_format=True)
 df_train.loc[:, 'Date'] = df_train.Date.dt.strftime("%m%d")
 df_train["Date"] = df_train["Date"].astype(int)
 df_test.loc[:, 'Date'] = df_test.Date.dt.strftime("%m%d")
 df_test["Date"] = df_test["Date"].astype(int )<data_type_conversions>
feats = [c for c in df.columns if c not in ['nota_mat','codigo_mun', 'Unnamed: 0']]
df_train['ConfirmedCases'] = df_train['ConfirmedCases'].apply(int)
 df_train['Fatalities'] = df_train['Fatalities'].apply(int )<data_type_conversions>
nme = df.groupby(['estado'], as_index=False ).mean()
for c in nme.columns:
nme[c].fillna(nme[c].mean() , inplace=True )
df_train['Province_State'] = df_train['Province_State'].fillna('unknown')
 df_test['Province_State'] = df_test['Province_State'].fillna('unknown' )<import_modules>
df2 = pd.merge(df, nme, left_on='estado', right_on='estado', how='left', suffixes=('', '_mean'))
df2['estado_mean']=df2['estado']
import xgboost as xgb
from xgboost import XGBRegressor<prepare_x_and_y>
feats2 = [c for c in feats if c not in ['codigo_mun']]
features = ['Date']
submission = pd.DataFrame(columns=['ForecastId', 'ConfirmedCases', 'Fatalities'])
 for i in tqdm(df_train.Country_Region.unique()):
c_train = df_train[df_train['Country_Region'] == i]
c_test = df_test[df_test['Country_Region'] == i]
for j in c_train.Province_State.unique() :
p_train = c_train[c_train['Province_State'] == j]
p_test = c_test[c_test['Province_State'] == j]
x_train = p_train[features]
y_train_cc = p_train['ConfirmedCases']
y_train_ft = p_train['Fatalities']
model = xgb.XGBRegressor(n_estimators=1000)
 model.fit(x_train, y_train_cc)
 y_pred_cc = model.predict(p_test[features])
 model.fit(x_train, y_train_ft)
 y_pred_ft = model.predict(p_test[features])
 p_test['ConfirmedCases'] = y_pred_cc
p_test['Fatalities'] = y_pred_ft
submission = pd.concat([submission, p_test[['ForecastId', 'ConfirmedCases', 'Fatalities']]], axis=0 )<save_to_csv>
for c in feats2:
df[c].fillna(df2[c+'_mean'], inplace=True )
submission.to_csv('submission.csv', index=False )<save_to_csv>
df['nota_mat'] = np.log(df['nota_mat'] )
submission.to_csv('submission.csv', index=False )<compute_test_metric>
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
%matplotlib inline
def sigmoid_sqrt_func(x, a, b, c, d, e):
return c + d /(1.0 + np.exp(-a*x+b)) + e*x**0.5
def sigmoid_linear_func(x, a, b, c, d, e):
return c + d /(1.0 + np.exp(-a*x+b)) + e*0.1*x
def sigmoid_quad_func(x, a, b, c, d, e, f):
return c + d /(1.0 + np.exp(-a*x+b)) + e*0.1*x + f*0.001*x*x
def sigmoid_func(x, a, b, c, d):
return c + d /(1.0 + np.exp(-a*x+b))
def exp_func(x, a, b, c, d):
return c + d * np.exp(a*x+b)
 def func_fitting(y, func=sigmoid_func, x_scale=50.0, y_scale=10000.0, start_pred=8, AN=0, MAXN=60, PN=15, b=5):
x = range(len(y))
x_real = np.array(x)/x_scale
y_real = np.array(y)/y_scale
x_train = x_real
y_train = y_real
def next_day_pred(AN, BN):
x_train = x_real[AN:BN]
y_train = y_real[AN:BN]
popt, pcov = curve_fit(func, x_train, y_train,
method='trf',
maxfev=20000,
p0=(1, 0, 0, 1),
bounds=[(-b, -np.inf, -np.inf, -b),(b, np.inf, np.inf, b)], 
)
 x_pred = np.array(range(MAXN)) /x_scale
y_pred = func(x_pred, *popt)
 return x_pred, y_pred
NP = start_pred
y_pred = [np.nan]*NP
y_pred_list = []
for BN in range(NP, len(y_real)) :
x_pred, y_pred_ = next_day_pred(BN-PN, BN)
 y_pred.append(y_pred_[BN])
 y_pred_list.append(y_pred_)
 for BN in range(len(y_real), len(y_pred_)) :
y_pred.append(y_pred_[BN])
 y_pred = np.array(y_pred)
 y_pred_list = np.array(y_pred_list)
 y_pred_std = np.std(y_pred_list[-2:], axis=0)
 return x_real*x_scale, y_real*y_scale, x_train*x_scale, y_train*y_scale, \
x_pred*x_scale, y_pred*y_scale, y_pred_std*y_scale
def draw_figure(start_date, title, x_real, y_real, x_train, y_train, x_pred, y_pred, y_pred_std):
def to_date(idx):
idx = np.round(idx)
 return datetime.datetime.strptime(start_date, '%m/%d/%Y' ).date() + datetime.timedelta(days=idx)
 fig, ax1 = plt.subplots(figsize=[14, 7])
 plot1 = ax1.plot(list(map(to_date, x_real)) , y_real, 'gs',label='original')
 plot2 = ax1.plot(list(map(to_date, x_pred)) , y_pred, 'r',label='predict')
 plot3 = ax1.fill_between(list(map(to_date, x_pred)) ,
np.maximum(0,(y_pred-y_pred_std)) , 
(y_pred+y_pred_std),
alpha=0.2, edgecolor='
plot0 = ax1.plot(list(map(to_date, x_train)) , y_train, 'y.',label='history')
 ax2=ax1.twinx()
ax2.plot(list(map(to_date, x_real)) [1:],(y_real[1:]-y_real[:-1]), '-s',label='original add')
 ax2.plot(list(map(to_date, x_pred)) [1:],(y_pred[1:]-y_pred[:-1]), '-',label='pred add')
 plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%m-%d'))
plt.gca().xaxis.set_major_locator(mdates.DayLocator())
 plt.gcf().autofmt_xdate()
plt.xlabel('x')
 plt.ylabel('y')
 fig.legend(loc=2)
 plt.title(title)
 plt.savefig('{}.pdf'.format(title))
plt.show()
date = list(map(to_date, x_pred))
pred = y_pred
real = y_real
for i in range(len(pred)) :
if i < len(real):
print('{}\t{:.0f}\t{:.0f}\t{:.3f}'.format(date[i], real[i], pred[i], np.abs(pred[i]-real[i])/real[i]*100))
else:
print('{}\t-\t{:.0f}'.format(date[i], pred[i]))
return pred<load_from_csv>
plt.rcParams['figure.figsize'] =(20,30 )
train_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/train.csv')
 test_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/test.csv')
 pred_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/submission.csv')
 train_data = train_data.fillna(value='NULL')
 test_data = test_data.fillna(value='NULL' )<count_unique_values>
test = df[df['nota_mat'].isnull() ]
train = df[df['nota_mat'].notnull() ]
train_date_list = train_data.iloc[:, 3].unique()
print(len(train_date_list))
print(train_date_list)
 test_date_list = test_data.iloc[:, 3].unique()
print(len(test_date_list))
print(test_date_list)
 len(train_data.groupby(['Province_State', 'Country_Region']))
len(test_data.groupby(['Province_State', 'Country_Region']))<load_from_csv>
from sklearn.model_selection import train_test_split
start_date = '01/22/2020'
start_pred = 74
start_submit = 64
len_pred = 30
test_date_list = test_data.iloc[:, 3].unique()
test_data_filled = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-3/test.csv')
 test_data_filled = test_data_filled.fillna(value='NULL')
 test_data_filled['ConfirmedCases'] = pred_data['ConfirmedCases']
test_data_filled['Fatalities'] = pred_data['Fatalities']
for idx,(k, v)in enumerate(train_data.groupby(['Province_State', 'Country_Region'])) :
print(idx, k, end=' ')
 b_cc, b_f = 5, 3
if k[1] == 'China':
b_cc = 1
b_f = 1
if k[0] == 'Hong Kong':
b_cc = 3
elif k[1] == 'Italy':
b_cc = 10
b_f = 3
elif k[1] == 'US':
b_cc = 5
b_f = 3
elif k[1] == 'Spain':
b_cc = 4
b_f = 3
hist_num = v.loc[:,'ConfirmedCases'].tolist()
ret = func_fitting(hist_num, y_scale=max(1000, np.max(hist_num)) , b=b_cc,
start_pred=start_pred, PN=10, MAXN=len(hist_num)+len_pred)
 ret = list(ret)
 real_cc = np.round(np.array(ret[1]))
pred_cc = np.round(np.array(ret[5]))
for i in range(len(real_cc)) :
pred_cc[i] = real_cc[i]
pred_cc = pred_cc[start_submit:]
print(pred_cc)
 hist_num = v.loc[:,'Fatalities'].tolist()
ret = func_fitting(hist_num, y_scale=max(1000, np.max(hist_num)) , b=b_f,
start_pred=start_pred, PN=10, MAXN=len(hist_num)+len_pred)
 ret = list(ret)
 real_f = np.round(np.array(ret[1]))
pred_f = np.round(np.array(ret[5]))
for i in range(len(real_f)) :
pred_f[i] = real_f[i]
pred_f = pred_f[start_submit:]
print(pred_f)
 for i in range(14, len(pred_cc)) :
if pred_f[i] < 20 and pred_cc[i] > 200 and k[1] != 'China':
pred_f[i] = pred_cc[i] * 0.01 * np.log10(pred_cc[i])
 print(pred_f)
 print(pred_cc[-1], pred_f[-1])
 for i in range(len(pred_cc)) :
index =(test_data_filled['Province_State'] == k[0])& \ 
(test_data_filled['Country_Region'] == k[1])& \ 
(test_data_filled['Date'] == test_date_list[i])
 test_data_filled.loc[index, 'ConfirmedCases'] = pred_cc[i]
test_data_filled.loc[index, 'Fatalities'] = pred_f[i]<filter>
from sklearn.model_selection import train_test_split
submission = test_data_filled.loc[:,['ForecastId', 'ConfirmedCases', 'Fatalities']]<save_to_csv>
train, valid = train_test_split(train, random_state = 42 )
submission.to_csv("submission.csv", index=False)
 submission.head(500 )<import_modules>
rf = RandomForestRegressor(random_state = 42)
 print('Parameters currently in use:
')
 pprint(rf.get_params() )
import pandas as pd
import numpy as np
from sklearn.linear_model import LogisticRegression
import matplotlib.pyplot as plt
from sklearn.utils import shuffle
from sklearn.model_selection import GridSearchCV<load_from_csv>
best = rf_random.best_params_
best2 = {'n_estimators': 800,
'min_samples_split': 2,
'min_samples_leaf': 2,
'max_features': 'sqrt',
'max_depth': 50,
'bootstrap': False}
best
tourney_result = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneyCompactResults.csv')
 tourney_seed = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MNCAATourneySeeds.csv' )<drop_column>
best_random = RandomForestRegressor(n_estimators= 800, min_samples_split = 2, min_samples_leaf= 2, max_features = 'sqrt', max_depth =50, bootstrap= False )
tourney_result = tourney_result.drop(['DayNum', 'WScore', 'LScore', 'WLoc', 'NumOT'], axis=1)
 tourney_result<merge>
def evaluate(model, test_features, test_labels):
predictions = model.predict(test_features)
 errors = abs(predictions - test_labels)
 mape = 100 * np.mean(errors / test_labels)
 accuracy = 100 - mape
print('Model Performance')
 print('Average Error: {:0.4f} degrees.'.format(np.mean(errors)))
 print('Accuracy = {:0.2f}%.'.format(accuracy))
return accuracy
tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Seed':'WSeed'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result = pd.merge(tourney_result, tourney_seed, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Seed':'LSeed'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result<feature_engineering>
base_model = RandomForestRegressor(n_estimators = 10, random_state = 42)
 base_model.fit(train[feats], train['nota_mat'])
 base_accuracy = evaluate(base_model, valid[feats], valid['nota_mat'] )
def get_seed(x):
return int(x[1:3])
 tourney_result['WSeed'] = tourney_result['WSeed'].map(lambda x: get_seed(x))
tourney_result['LSeed'] = tourney_result['LSeed'].map(lambda x: get_seed(x))
tourney_result<load_from_csv>
best_random.fit(train[feats], train['nota_mat'])
 random_accuracy = evaluate(best_random, valid[feats], valid['nota_mat'] )
season_result = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-mens-tournament/MDataFiles_Stage1/MRegularSeasonCompactResults.csv' )<concatenate>
print('Improvement of {:0.2f}%.'.format(100 *(random_accuracy - base_accuracy)/ base_accuracy))
season_win_result = season_result[['Season', 'WTeamID', 'WScore']]
season_lose_result = season_result[['Season', 'LTeamID', 'LScore']]
season_win_result.rename(columns={'WTeamID':'TeamID', 'WScore':'Score'}, inplace=True)
 season_lose_result.rename(columns={'LTeamID':'TeamID', 'LScore':'Score'}, inplace=True)
 season_result = pd.concat(( season_win_result, season_lose_result)).reset_index(drop=True)
 season_result<groupby>
test['nota_mat'] = best_random.predict(test[feats] )
season_score = season_result.groupby(['Season', 'TeamID'])['Score'].sum().reset_index()
season_score<merge>
test['nota_mat'] = np.exp(test['nota_mat'])
 test['codigo_mun'] = test['codigo_mun'].apply(lambda x: x.replace('ID_ID_', ''))
tourney_result = pd.merge(tourney_result, season_score, left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Score':'WScoreT'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result = pd.merge(tourney_result, season_score, left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], how='left')
 tourney_result.rename(columns={'Score':'LScoreT'}, inplace=True)
 tourney_result = tourney_result.drop('TeamID', axis=1)
 tourney_result<drop_column>
test[['codigo_mun','nota_mat']].to_csv('rf.csv', index=False )