Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<groupby>
%matplotlib inline
def transform_dataset(ds):
transformed_dataset = ds.copy()
transformed_dataset['Age'].fillna(transformed_dataset['Age'].median() , inplace=True)
 transformed_dataset['Fare'].fillna(transformed_dataset['Fare'].median() , inplace=True)
 transformed_dataset['Sex'] = pd.factorize(transformed_dataset['Sex'])[0]
transformed_dataset=pd.concat([transformed_dataset, pd.get_dummies(transformed_dataset['Embarked'], prefix="Embarked")], axis=1)
 transformed_dataset['Childs'] = transformed_dataset.apply(lambda row: get_childs(row), axis=1)
 transformed_dataset['Parents'] = transformed_dataset.apply(lambda row: get_parents(row), axis=1)
 transformed_dataset = transformed_dataset.drop(['PassengerId', 'Name', 'Ticket', 'Cabin', 'Embarked', 'Parch'], axis=1)
 return transformed_dataset
def get_childs(row):
if(row.Parch>2 or row.Age>=18):
return row.Parch
return 0
def get_parents(row):
if(row.Parch <= 2 and row.Age<18):
return row.Parch
return 0
test_country_df = test_df.groupby(['Date', 'Country'])[['ConfirmedCases', 'Fatalities']].sum().reset_index()
display(test_country_df[test_country_df['Country']=='Australia'][:20])
 for country in [x for x in province_countries if x in top30_countries]:
present_country_df = test_country_df[test_country_df['Country']==country].reset_index()
present_country_df['prev_cases'] = present_country_df.groupby('Country')['ConfirmedCases'].shift(1)
 present_country_df['New Case'] = present_country_df['ConfirmedCases'] - present_country_df['prev_cases']
present_country_df['New Case'].fillna(0, inplace=True)
 present_country_df['prev_deaths'] = present_country_df.groupby('Country')['Fatalities'].shift(1)
 present_country_df['New Death'] = present_country_df['Fatalities'] - present_country_df['prev_deaths']
present_country_df['New Death'].fillna(0, inplace=True)
 px.bar(present_country_df,
x='Date', y='New Case', color='Country',
title=f'DAILY NEW Confirmed cases in '+country ).show() 
<feature_engineering>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv')
 passenger_id = test['PassengerId']
for country in no_province_countries:
current_considered_country_df = no_province_country_dfs[country][['ConfirmedCases','Fatalities','Days']].reset_index()
print(country)
 for i in range(train_end_day-test_start_day+1):
test_df.loc[(test_df['Country']==country)&(test_df['Days']==i+test_start_day), 'ConfirmedCases'] = current_considered_country_df.loc[current_considered_country_df['Days'] == i+test_start_day, 'ConfirmedCases'].values[0]
test_df.loc[(test_df['Country']==country)&(test_df['Days']==i+test_start_day), 'Fatalities'] = current_considered_country_df.loc[current_considered_country_df['Days'] == i+test_start_day, 'Fatalities'].values[0]
indexNames = current_considered_country_df[ current_considered_country_df['ConfirmedCases'] == 0 ].index
current_considered_country_df.drop(indexNames , inplace=True)
 cases_train = np.diff(current_considered_country_df['ConfirmedCases'].to_numpy())
 fatalities_train = np.diff(current_considered_country_df['Fatalities'].to_numpy())
 fatal_rate = 0.0
if(current_considered_country_df['Fatalities'].to_numpy() [-1]>0):
fatal_rate = current_considered_country_df['Fatalities'].to_numpy() [-1]/current_considered_country_df['ConfirmedCases'].to_numpy() [-1]
print("fatal rate is: "+str(fatal_rate))
cases_increase_avg = 0
days=0
for i in range(len(cases_train)-1):
cases_increase_avg +=(cases_train[i+1] -cases_train[i])
 days+=1
if(days>0):
cases_increase_avg = int(cases_increase_avg/days)
 fatal_increase_avg = 0
days=0
for i in range(len(fatalities_train)-1):
fatal_increase_avg +=(fatalities_train[i+1] -fatalities_train[i])
 days+=1
if(days>0):
fatal_increase_avg = int(fatal_increase_avg/days)
 n_steps = max(int(len(cases_train)*0.15),2)
 avg_weekly_per_day_case = []
avg_window = 4
avg_step = 2
X_cases_val, y_cases_val = split_sequence(cases_train,n_steps)
 X_cases_val = np.reshape(X_cases_val,(X_cases_val.shape[0],X_cases_val.shape[1],1))
model_cases=build_model(n_steps)
 model_cases.fit(X_cases_val, y_cases_val, epochs=50, verbose=0)
 cases_predict_next = model_cases.predict(np.reshape(X_cases_val[len(X_cases_val)-1],(1,n_steps,1)) , verbose=0 ).astype(int)
 cases_predict_next[0] = np.array([max(0,cases_predict_next[0])])
 X_fatal_val, y_fatal_val = split_sequence(fatalities_train,n_steps)
 X_fatal_val = np.reshape(X_fatal_val,(X_fatal_val.shape[0],X_fatal_val.shape[1],1))
model_fatalities=build_model(n_steps)
 model_fatalities.fit(X_fatal_val, y_fatal_val, epochs=50, verbose=0)
 fatality_predict_next = model_fatalities.predict(np.reshape(X_fatal_val[len(X_fatal_val)-1],(1,n_steps,1)) , verbose=0 ).astype(int)
 fatality_predict_next[0] = np.array([max(0,fatality_predict_next[0])])
 fatality_predict_next[0] = np.array([max(fatality_predict_next[0],cases_predict_next[0]*fatal_rate)])
 test_df.loc[(test_df['Country']==country)&(test_df['Days']==train_end_day+1), 'ConfirmedCases'] = test_df.loc[(test_df['Country']==country)&(test_df['Days']==train_end_day), 'ConfirmedCases'].values[0] + cases_predict_next[0]
test_df.loc[(test_df['Country']==country)&(test_df['Days']==train_end_day+1), 'Fatalities'] = test_df.loc[(test_df['Country']==country)&(test_df['Days']==train_end_day), 'Fatalities'].values[0] + fatality_predict_next[0]
new_entry_cases=X_cases_val[len(X_cases_val)-1]
new_entry_fatal=X_fatal_val[len(X_fatal_val)-1]
for i in range(test_end_day-train_end_day-1):
last_series = np.reshape(new_entry_cases,(1,n_steps,1))
new_entry_cases=np.delete(last_series,[0])
 new_entry_cases = np.insert(new_entry_cases,n_steps-1,cases_predict_next[0])
 cases_predict_next = model_cases.predict(np.reshape(new_entry_cases,(1,n_steps,1)) , verbose=0 ).astype(int)
 if(cases_predict_next[0]-new_entry_cases[n_steps-1]>cases_increase_avg):
cases_predict_next[0] = np.array([max(0,new_entry_cases[n_steps-1]+cases_increase_avg)])
 cases_predict_next[0] = np.array([max(0,cases_predict_next[0])])
 last_series = np.reshape(new_entry_fatal,(1,n_steps,1))
new_entry_fatal=np.delete(last_series,[0])
 new_entry_fatal = np.insert(new_entry_fatal,n_steps-1,fatality_predict_next[0])
 fatality_predict_next = model_fatalities.predict(np.reshape(new_entry_fatal,(1,n_steps,1)) , verbose=0 ).astype(int)
 if(fatality_predict_next[0]-new_entry_fatal[n_steps-1]>fatal_increase_avg):
fatality_predict_next[0] = max(0,new_entry_fatal[n_steps-1]+fatal_increase_avg)
 fatality_predict_next[0] = np.array([max(0,fatality_predict_next[0])])
 fatality_predict_next[0] = np.array([max(fatality_predict_next[0],int(cases_predict_next[0]*fatal_rate)) ])
 test_df.loc[(test_df['Country']==country)&(test_df['Days']==i+train_end_day+2), 'ConfirmedCases'] = test_df.loc[(test_df['Country']==country)&(test_df['Days']==i+train_end_day+1), 'ConfirmedCases'].values[0] + cases_predict_next[0]
test_df.loc[(test_df['Country']==country)&(test_df['Days']==i+train_end_day+2), 'Fatalities'] = test_df.loc[(test_df['Country']==country)&(test_df['Days']==i+train_end_day+1), 'Fatalities'].values[0] + fatality_predict_next[0]
del model_fatalities
del model_cases<save_to_csv>
transformed_train = transform_dataset(train)
 transformed_test = transform_dataset(test )
test_df_copy = test_df
submission_df_copy = submission_df
submit = pd.DataFrame()
submit['ForecastId'] = test_df['ForecastId']
submit['ConfirmedCases'] = test_df['ConfirmedCases']
submit['Fatalities'] = test_df['Fatalities']
submit = submit.reset_index()
submit = submit.drop(['Date'], axis=1)
 display(submit.tail())
 submit.to_csv('submission.csv',index=False )<set_options>
X = transformed_train.drop(['Survived'], axis=1)
 Y = transformed_train['Survived']
pd.options.display.max_rows = 500
pd.options.display.max_columns = 500
%matplotlib inline 
<load_from_csv>
X_train, X_test, Y_train, Y_test = model_selection.train_test_split(X, Y, test_size = 0.2, random_state = 21 )
train = pd.read_csv('.. /input/covid19-global-forecasting-week-4/train.csv')
 us_before = pd.read_csv('.. /input/jhu-covid19-data-with-us-state-data-prior-to-mar-9/covid19_train_data_us_states_before_march_09_new.csv')
 update =(train['Country_Region'] == 'US')&(train.Date <= '2020-03-09')
 df = train[update]
us_before = us_before[(us_before['Country.Region'] == 'US')&(us_before.Date <= '2020-03-09')]
us_before = us_before[['Province.State', 'Country.Region', 'Date', 'ConfirmedCases', 'Fatalities']]
us_before.columns = ['Province_State', 'Country_Region', 'Date', 'ConfirmedCases_new', 'Fatalities_new']
df.shape, us_before.shape
df = df.merge(us_before, how='left', on=['Province_State', 'Country_Region', 'Date'] ).fillna(0)
 train.loc[update, 'ConfirmedCases'] = df['ConfirmedCases_new'].values
train.loc[update, 'Fatalities'] = df['Fatalities_new'].values
train['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 train['day'] = train.Date.dt.dayofyear
train['geo'] = ['_'.join(x)for x in zip(train['Country_Region'], train['Province_State'])]
train
delta =(train[train.geo == 'France_'].ConfirmedCases.diff().max())/ 2
train.loc[10405:10414, 'ConfirmedCases'] = train.loc[10405:10414, 'ConfirmedCases'] - delta
test = pd.read_csv('.. /input/covid19-global-forecasting-week-4/test.csv')
 test['Province_State'].fillna('', inplace=True)
 test['Date'] = pd.to_datetime(test['Date'])
 test['day'] = test.Date.dt.dayofyear
test['geo'] = ['_'.join(x)for x in zip(test['Country_Region'], test['Province_State'])]
test
day_min = train['day'].min()
train['day'] -= day_min
test['day'] -= day_min
min_test_val_day = test.day.min()
max_test_val_day = train.day.max()
max_test_day = test.day.max()
num_days = max_test_day + 1
min_test_val_day, max_test_val_day, num_days
train['ForecastId'] = -1
test['Id'] = -1
test['ConfirmedCases'] = 0
test['Fatalities'] = 0
data = pd.concat([train,
test[test.day > max_test_val_day][train.columns]
] ).reset_index(drop=True)
 dates = data[data['geo'] == 'France_'].Date.values
region_meta = pd.read_csv('.. /input/covid19-forecasting-metadata/region_metadata.csv')
 region_meta['Province_State'].fillna('', inplace=True)
 region_meta['geo'] = ['_'.join(x)for x in zip(region_meta['Country_Region'], region_meta['Province_State'],)]
region_meta
set(data.geo.unique())- set(region_meta.geo.unique())
 region_meta = data[['geo']].merge(region_meta, how='left', on='geo')
 region_meta = region_meta.groupby('geo' ).first()
population = np.log1p(region_meta[['population']])
 population = population[['population']].values
population.shape
lockdown_date = pd.read_csv('.. /input/covid19-lockdown-dates-by-country/countryLockdowndates.csv')
 lockdown_date['Province'].fillna('', inplace=True)
 lockdown_date['Date'] = pd.to_datetime(lockdown_date['Date'], dayfirst=True)
 lockdown_date['lock_day'] = lockdown_date.Date.dt.dayofyear
lockdown_date['lock_day'] -= day_min
lockdown_date['geo'] = ['_'.join(x)for x in zip(lockdown_date['Country/Region'], lockdown_date['Province'])]
lockdown_date
lockdown_date = data[['geo', 'day']].merge(lockdown_date[['geo', 'lock_day']], how='left', on=['geo'])
 lockdown_date['locked'] = 1 *(lockdown_date['day'] > lockdown_date['lock_day'])
 lockdown_date = lockdown_date.pivot(index='geo', columns='day', values='locked' ).values
lockdown_date
geo_data = data.pivot(index='geo', columns='day', values='ForecastId')
 num_geo = geo_data.shape[0]
geo_data
geo_id = {}
for i,g in enumerate(geo_data.index):
geo_id[g] = i
num_geo
ConfirmedCases = data.pivot(index='geo', columns='day', values='ConfirmedCases')
 Fatalities = data.pivot(index='geo', columns='day', values='Fatalities')
 cases = np.log1p(ConfirmedCases.values)
 deaths = np.log1p(Fatalities.values)
 def get_dataset(start_pred, num_train, lag_period, continents_ids_base, country_ids_base, cases, deaths,
population, time_cases, time_deaths, lockdown_date):
days = np.arange(start_pred - num_train + 1, start_pred + 1)
 lag_cases = np.vstack([cases[:, d - lag_period : d] for d in days])
 lag_deaths = np.vstack([deaths[:, d - lag_period : d] for d in days])
 target_cases = np.vstack([cases[:, d : d + 1] for d in days])
 target_deaths = np.vstack([deaths[:, d : d + 1] for d in days])
 continents_ids = np.vstack([continents_ids_base for d in days])
 country_ids = np.vstack([country_ids_base for d in days])
 population = np.vstack([population for d in days])
 time_case = np.vstack([time_cases[:, d - 1: d ] for d in days])
 time_death = np.vstack([time_deaths[:, d - 1 : d ] for d in days])
 lockdown = [get_lockdown(lockdown_date, d)for d in days]
lockdown_case = np.vstack([l[0] for l in lockdown])
 lockdown_death = np.vstack([l[1] for l in lockdown])
 start_pred = np.hstack([d * np.ones(( cases.shape[0],)) for d in days] ).astype('int')
 return(lag_cases, lag_deaths, target_cases, target_deaths,
continents_ids, country_ids, population, time_case, time_death,
lockdown_case, lockdown_death, start_pred, days)
 def update_valid_dataset(data, pred_death, pred_case, cases, deaths): 
(lag_cases, lag_deaths, target_cases, target_deaths,
continents_ids, country_ids, population, time_case, time_death,
lockdown_case, lockdown_death, start_pred, days)= data
day = days[-1] + 1
new_lag_cases = np.hstack([lag_cases[:, 1:], pred_case])
 new_lag_deaths = np.hstack([lag_deaths[:, 1:], pred_death])
 new_target_cases = cases[:, day:day+1]
new_target_deaths = deaths[:, day:day+1]
new_continents_ids = continents_ids
new_country_ids = country_ids
new_population = population
new_time_death, new_time_case = update_time(time_death, time_case, pred_death, pred_case)
 new_lockdown_case = lockdown_case
new_lockdown_death = lockdown_death
new_start_pred = 1 + start_pred
new_days = 1 + days
return(new_lag_cases, new_lag_deaths, new_target_cases, new_target_deaths,
new_continents_ids, new_country_ids, new_population,
new_time_case, new_time_death, new_lockdown_case, new_lockdown_death,
new_start_pred, new_days)
 def train_model(train, valid, start_lag_death, end_lag_death, num_lag_case, num_val, score, cases, deaths,):
alpha = 3
lr_death = Ridge(alpha=alpha, fit_intercept=True)
 lr_case = Ridge(alpha=alpha, fit_intercept=False)
(train_death_score, train_case_score, train_pred_death, train_pred_case, 
)= fit_eval(lr_death, lr_case, train, start_lag_death, end_lag_death, num_lag_case, fit=True, score=score,
cases=cases, deaths=deaths)
 death_scores = []
case_scores = []
death_pred = []
case_pred = []
for i in range(num_val): 
(valid_death_score, valid_case_score, valid_pred_death, valid_pred_case, 
)= fit_eval(lr_death, lr_case, valid, start_lag_death, end_lag_death, num_lag_case,
fit=False, score=score,
cases=cases, deaths=deaths)
 death_scores.append(valid_death_score)
 case_scores.append(valid_case_score)
 death_pred.append(valid_pred_death)
 case_pred.append(valid_pred_case)
 if 0:
print('val death: %0.3f' % valid_death_score,
'val case: %0.3f' % valid_case_score,
'val : %0.3f' % np.mean([valid_death_score, valid_case_score]),
flush=True)
 valid = update_valid_dataset(valid, valid_pred_death, valid_pred_case, cases, deaths)
 if score:
death_scores = np.sqrt(np.mean([s**2 for s in death_scores]))
case_scores = np.sqrt(np.mean([s**2 for s in case_scores]))
if 0:
print('train death: %0.3f' % train_death_score,
'train case: %0.3f' % train_case_score,
'val death: %0.3f' % death_scores,
'val case: %0.3f' % case_scores,
'val : %0.3f' %(( death_scores + case_scores)/ 2),
flush=True)
 else:
print('%0.4f' % case_scores,
', %0.4f' % death_scores,
'= %0.4f' %(( death_scores + case_scores)/ 2),
flush=True)
 death_pred = np.hstack(death_pred)
 case_pred = np.hstack(case_pred)
 return death_scores, case_scores, death_pred, case_pred
def get_country_ids(last_train, case_threshold):
countries = [g.split('_')[0] for g in geo_data.index]
countries = pd.factorize(countries)[0]
countries[cases[:, :last_train+1].max(axis=1)< np.log1p(case_threshold)] = -1
countries = pd.factorize(countries)[0]
country_ids_base = countries.reshape(( -1, 1))
ohe = OneHotEncoder(sparse=False)
 country_ids_base = 0.2 * ohe.fit_transform(country_ids_base)
 return country_ids_base
def get_continent_ids() :
continents = region_meta['continent']
continents = pd.factorize(continents)[0]
continents_ids_base = continents.reshape(( -1, 1))
ohe = OneHotEncoder(sparse=False)
 continents_ids_base = ohe.fit_transform(continents_ids_base)
 return continents_ids_base
def val_score(true, pred):
pred = np.log1p(np.round(np.expm1(pred)- 0.2))
return np.sqrt(mean_squared_error(true.ravel() , pred.ravel()))
def val_score(true, pred):
return np.sqrt(mean_squared_error(true.ravel() , pred.ravel()))
def predict(tdata, num_lag, eps, w, window):
num_pred = 1
num_geo = tdata.shape[0]
days = np.arange(num_lag)
 pred = np.zeros(( num_geo, num_pred))
start_pred = tdata.shape[1]
x0 = np.arange(start_pred ).reshape(( -1, 1))
w_delta =(window - 1)/2
x1 = np.arange(start_pred + w_delta, start_pred + 1 + w_delta ).reshape(( -1, 1))
w0 = np.array([w** i for i in range(num_lag)])
 w0 = w0 / w0.mean()
for i in range(num_geo):
y0 = pd.Series(tdata[i, :])
 y = y0.rolling(window=window ).mean().fillna(method='bfill')
 y = y.diff().fillna(0)
 y = np.clip(y.values, 0, 1e6)
 if y.max() == 0:
pred[i, :] = 0
continue
x = x0
filter_ =(y > 0)
 x = x[y > 0]
y = y[y > 0]
y = y[-num_lag:]
x = x[-num_lag:]
pad = num_lag - len(y)
 if pad > 0:
y = np.hstack(( np.zeros(pad), y))
xmin = x.min()
x = np.vstack(( np.arange(xmin - pad, xmin ).reshape(( -1, 1)) , x))
w = w0[-len(y):]
if len(y)< num_lag:
pred[i, :] = 0
continue
y = np.log(y + eps)
 lr = Ridge(fit_intercept=True)
 lr.fit(x, y, w)
 if 0 and lr.coef_[0] > 0:
print('up', lr.coef_, lr.intercept_)
 y_pred = lr.predict(x1)
 y_pred = np.clip(y_pred, -10, np.average(y, weights=w))
y_pred = np.exp(y_pred)- eps
y_pred = np.clip(y_pred, 0, 5)
 y_pred = np.cumsum(y_pred)
 pred[i, :] = y_pred
return pred
def fit_eval(lr_death, lr_case, data, start_lag_death, end_lag_death, num_lag_case, fit, score, cases, deaths): 
(lag_cases, lag_deaths, target_cases, target_deaths,
continents_ids, country_ids, population, time_case, time_death,
lockdown_case, lockdown_death, start_pred, days)= data
idx = np.arange(lag_cases.shape[0])
 X_trend0 = predict(lag_deaths[:, -lag_period:], num_lag_death_trend, eps_trend, w_trend, window_trend)
 X_trend = predict(lag_cases[:, -lag_period-lag_case_death:-lag_case_death], num_lag_death_trend, eps_trend, w_trend, window_trend)
 y_death_prev = lag_deaths[:, -1:]
X_trend0 = X_trend0 + y_death_prev
X_trend = X_trend + y_death_prev
w_t = 0.4 *(lag_deaths[:, -num_lag_death_trend:1-num_lag_death_trend] > np.log1p(death_trend_thr))
X_trend = w_t * X_trend0 +(1 - w_t)* X_trend
y_death = target_deaths
if 1:
X_death = np.hstack([lag_cases[:, -start_lag_death:-end_lag_death],
lag_deaths[:, -num_lag_death:],
country_ids,
continents_ids,
population,
])
 if fit:
lr_death.fit(X_death, y_death)
 y_pred_death = lr_death.predict(X_death)
 if 0:
w_t = lag_deaths[:, -1:]
w_t =(0 + w_t)/(0 + np.max(w_t))
w_t = w_t**0.7
w_t = 0.6 * w_t *(lag_deaths[:, -num_lag_death_trend:1-num_lag_death_trend] > np.log1p(death_trend_thr))
w_t = 0.4 *(lag_deaths[:, -num_lag_death_trend:1-num_lag_death_trend] > np.log1p(death_trend_thr))
w_t = 0.6
y_pred_death =(( 1 - w_t)* y_pred_death + w_t * X_trend)
 else:
y_pred_death = X_trend
y_pred_death = np.maximum(y_pred_death, y_death_prev)
 y_case_prev = lag_cases[:, -1:]
X_trend = predict(lag_cases[:, -lag_period:], num_lag_trend, eps_trend, w_trend, window_trend)
 X_trend = X_trend + y_case_prev
X_case = np.hstack([lag_cases[:, -num_lag_case:],
country_ids,
continents_ids,
population,
time_case,
lockdown_case[idx, start_pred].reshape(( -1, 1)) ,
])
 y_case = target_cases
if fit:
lr_case.fit(X_case, y_case)
 y_pred_case = lr_case.predict(X_case)
 w_t = lag_cases[:, -1:]
w_t =(0 + w_t)/(0 + np.max(w_t))
w_t = np.clip(1.1 * w_t**0.7, 0, 1)
 y_pred_case =(( 1 - w_t)* y_pred_case + w_t * X_trend)
 y_pred_case = np.maximum(y_pred_case, y_case_prev)
 if score:
death_score = val_score(y_death, y_pred_death)
 case_score = val_score(y_case, y_pred_case)
 else:
death_score = 0
case_score = 0
return death_score, case_score, y_pred_death, y_pred_case
def update_time(time_death, time_case, pred_death, pred_case):
new_time_death = np.expm1(time_death)+ c_death *(pred_death >= np.log1p(t_death))
new_time_death = 1 *np.log1p(new_time_death)
 new_time_case = np.expm1(time_case)+ c_case *(pred_case >= np.log1p(t_case))
new_time_case = 1 *np.log1p(new_time_case)
 return new_time_death, new_time_case
c_case = 1
t_case = 1000
c_death = 1
t_death = 100
time_cases = c_case *(cases >= np.log1p(t_case))
time_cases = np.cumsum(time_cases, axis=1)
 time_cases = 1 * np.log1p(time_cases)
 time_cases.shape
time_deaths = c_death *(cases >= np.log1p(t_death))
time_deaths = np.cumsum(time_deaths, axis=1)
 time_deaths = 1 *np.log1p(time_deaths)
 time_deaths.shape
def get_lockdown(lockdown_date, day):
lockdown = lockdown_date.copy()
lockdown[:, day:] = 0
time_since_lockdown = np.cumsum(lockdown, axis=1)
 lockdown_cases = np.log1p(1e1*np.clip(time_since_lockdown - 12, 0, 100))
lockdown_deaths = np.log1p(1e1*np.clip(time_since_lockdown- 14, 0, 100))
return lockdown_cases, lockdown_deaths
start_lag_death, end_lag_death = 15, 5,
num_train = 7
num_lag_death = 6
num_lag_case = 12
lag_period = 15
lag_all_period = 30
case_threshold = 30
lag_case_death = 7 
(num_lag_trend, eps_trend, w_trend, window_trend)=(12, 1e-5, 1.2, 5)
 num_lag_death_trend = 12
death_trend_thr = 1
case_trend_thr = 1
scores = [] 
<create_dataframe>
param_grid =[ {'n_estimators' : [4,5,10, 15, 20, 25, 30, 35, 40], 'max_depth' : [5,10,15, 20]},]
rf = ensemble.GradientBoostingClassifier(random_state=21)
 model = GridSearchCV(rf,param_grid, cv = 5 )
def get_sub(start_val_delta=0):
start_val = min_test_val_day + start_val_delta
last_train = start_val - 1
num_val = max_test_val_day - start_val + 1
first_train = last_train + 1 -(num_train)
 keep_cases = cases
keep_deaths = deaths
print(dates[last_train], '%3d %3d' %(start_val, num_val), end=' ')
 country_ids_base = get_country_ids(last_train, case_threshold)
 continents_ids_base = get_continent_ids()
train_data = get_dataset(last_train, num_train, lag_period,
continents_ids_base, country_ids_base, keep_cases, keep_deaths,
population, time_cases, time_deaths, lockdown_date)
 valid_data = get_dataset(start_val, 1, lag_period,
continents_ids_base, country_ids_base, keep_cases, keep_deaths,
population, time_cases, time_deaths, lockdown_date)
 _, _, val_death_preds, val_case_preds = train_model(train_data, valid_data, start_lag_death, end_lag_death,
num_lag_case, num_val, True, keep_cases, keep_deaths, 
)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
sub = test[['Date', 'ForecastId', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub.fillna(0)
 sub = sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
return sub 
<merge>
model.fit(X_train,Y_train)
 print('train score = ', model.score(X_train,Y_train), '
test score = ', model.score(X_test,Y_test), '
', model.best_params_ )
known_test = train[['geo', 'day', 'ConfirmedCases', 'Fatalities']
].merge(test[['geo', 'day', 'ForecastId']], how='left', on=['geo', 'day'])
 known_test = known_test[['ForecastId', 'ConfirmedCases', 'Fatalities']][known_test.ForecastId.notnull() ].copy()
known_test
unknow_test = test[test.day > max_test_val_day]
unknow_test
def get_final_sub() :
start_val = max_test_val_day + 1
last_train = start_val - 1
num_val = max_test_day - start_val + 1
first_train = last_train + 1 -(num_train)
 keep_cases = cases
keep_deaths = deaths
print(dates[last_train], '%3d %3d' %(start_val, num_val), end=' ')
 country_ids_base = get_country_ids(last_train, case_threshold)
 continents_ids_base = get_continent_ids()
train_data = get_dataset(last_train, num_train, lag_period,
continents_ids_base, country_ids_base, keep_cases, keep_deaths,
population, time_cases, time_deaths, lockdown_date)
 valid_data = get_dataset(start_val, 1, lag_period,
continents_ids_base, country_ids_base, keep_cases, keep_deaths,
population, time_cases, time_deaths, lockdown_date)
 _, _, val_death_preds, val_case_preds = train_model(train_data, valid_data, start_lag_death, end_lag_death,
num_lag_case, num_val, True, keep_cases, keep_deaths, 
)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
print(unknow_test.shape, pred_deaths.shape, pred_cases.shape)
 sub = unknow_test[['Date', 'ForecastId', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
sub = pd.concat([known_test, sub])
 sub['ForecastId'] = sub['ForecastId'].astype('int')
 return sub
sub = get_final_sub()<save_to_csv>
Y_predict = model.predict(transformed_test)
 Y_p = pd.DataFrame(Y_predict, columns=['Survived'])
 Y_p
sub.to_csv('submission.csv', index=None )<merge>
res = pd.concat([passenger_id, Y_p], axis=1)
 res
<set_options><EOS> 
res.to_csv('res.csv', index=None )