Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<merge>
warnings.filterwarnings('ignore')
 RS=81
test_df = pd.merge(test_df, metadata_df, how='left')
 test_df = pd.merge(test_df, country_df, how='left')
 test_df = pd.merge(test_df, testinfo, how='left', left_on=group_cols, right_on=group_cols)
 test_df = pd.merge(test_df, fatalities, how='left')
 test_df = pd.merge(test_df, first_confirmed, how='left')
 test_df = pd.merge(test_df, first_50, how='left')
 test_df = pd.merge(test_df, first_hundred, how='left' )<data_type_conversions>
train = pd.read_csv('.. /input/titanic/train.csv')
 test = pd.read_csv('.. /input/titanic/test.csv' )
for df in [train_df, test_df]:
df['Province_State'].fillna(missed, inplace=True)
 df['Province_State'] = df.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : State(x['Province_State'], x['Country_Region']), axis=1)
 df.loc[:, 'Date'] = df.Date.dt.strftime("%m%d")
 df["Date"] = df["Date"].astype(int )<categorify>
data=pd.concat([train, test], sort = True ).reset_index(drop=True)
 data.info()
le = preprocessing.LabelEncoder()
for df in [train_df, test_df]:
df['Country_Region'] = le.fit_transform(df['Country_Region'])
 df['Province_State'] = le.fit_transform(df['Province_State'] )<prepare_x_and_y>
def replace_titles(x):
title=x['Title']
if title in ['Don', 'Major', 'Capt', 'Jonkheer', 'Rev', 'Col', 'Sir']:
return 'Mr'
elif title in ['Countess', 'Mme', 'Lady', 'Dona']:
return 'Mrs'
elif title in ['Mlle', 'Ms']:
return 'Miss'
elif title =='Dr':
if x['Sex']=='Male':
return 'Mr'
else:
return 'Mrs'
else:
return title
data['Title'] = data['Name'].str.extract('([A-Za-z]+)\.', expand=False)
 data['Title']=data.apply(replace_titles, axis=1 )
sub = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 features = ['Country_Region', 'Province_State', 'Date']
for country in range(len(countries)) :
country_train = train_df.loc[train_df['Country_Region'] == countries[country]]
country_test = test_df.loc[test_df['Country_Region'] == countries[country]]
states = train_df.loc[train_df['Country_Region'] == countries[country],:].Province_State.unique()
for state in states:
train = country_train.loc[country_train['Province_State'] == state]
X_train = train[features].to_numpy()
y1_train = train[['ConfirmedCases']].to_numpy()
y2_train = train[['Fatalities']].to_numpy()
test = country_test.loc[country_test['Province_State'] == state]
X_test = test[features].to_numpy()
model1 = DecisionTreeClassifier()
model1.fit(X_train, y1_train)
 pred1 = np.round(model1.predict(X_test))
model2 = DecisionTreeClassifier()
model2.fit(X_train, y2_train)
 pred2 = np.round(model2.predict(X_test))
country_test_Id = test.loc[:, 'ForecastId']
pred = pd.DataFrame({'ForecastId': country_test_Id, 'ConfirmedCases': pred1, 'Fatalities': pred2})
 sub = pd.concat([sub, pred], axis=0 )<save_to_csv>
data['Age']=data.groupby(['Title', 'Pclass'])['Age'].apply(lambda x: x.fillna(x.median()))
data['Fare'].fillna(data['Fare'].median() , inplace=True )
sub.ForecastId = sub.ForecastId.astype('int')
 sub.to_csv('submission.csv', index=False )<import_modules>
data['Family_Size']=data['SibSp']+data['Parch']
data['Alone']=[1 if x==0 else 0 for x in data['SibSp']+data['Parch']]
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import sys
import os
import time
from tqdm import tqdm
from tqdm.keras import TqdmCallback
from datetime import timedelta
from copy import deepcopy
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Input, LSTM, Dense, Activation, Dropout
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau 
<define_variables>
data['Surname'] =data.Name.str.extract(r'([A-Za-z]+),', expand=False)
 df=data[data.duplicated(['Ticket','Surname'], keep=False)]
dfg=df.groupby(['Ticket','Surname'])[['Survived']].max().fillna(0.5)
 data=pd.merge(data, dfg.rename(columns={'Survived':'FamSurvived'}), how='left', on=['Ticket','Surname'])
 data['FamSurvived'].fillna(0.5, inplace=True )
train_path = '.. /input/covid19-global-forecasting-week-4/train.csv'
test_path = '.. /input/covid19-global-forecasting-week-4/test.csv'
sub_path = '.. /input/covid19-global-forecasting-week-4/submission.csv'<load_from_csv>
data['Age_T']=PowerTransformer().fit_transform(data[['Age']])
 data['Fare_T']=PowerTransformer().fit_transform(data[['Fare']] )
df = pd.read_csv('.. /input/covid19-global-forecasting-week-4/train.csv')
 df.tail()<load_from_csv>
data = pd.get_dummies(data, columns=['Title'], drop_first=True )
def load_csv(path):
df = pd.read_csv(path)
 df.fillna('None', inplace=True)
 return df<categorify>
train = data[:len(train)]
test = data[len(train):].drop(['Survived'],axis=1 )
def timesteps(data, steps):
results = []
for i in range(len(data)- steps):
results.append(data[i:i+steps+1].values.tolist())
 return np.array(results)
 def input_reshape(data, shape):
return data.reshape(shape)
 def slide1_window(data, value):
data = data.reshape(-1, 2 ).tolist()
new_data = data[1:]
new_data.append(value)
 return np.array(new_data)
 def fit_lstm(train, vali, lstm_input_shape=None, batch_size=1, epochs=1, verbose=1):
X_train, y_train = train[:, :-1], train[:, -1]
X_train = input_reshape(X_train, lstm_input_shape)
 vali_data =(input_reshape(vali[:, :-1], lstm_input_shape), vali[:, -1])
 model = Sequential()
model.add(LSTM(128, input_shape=(lstm_input_shape[1:])))
 model.add(Dropout(0.2))
model.add(Dense(lstm_input_shape[-1]))
early_stopping = EarlyStopping(monitor='val_loss', patience=20)
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', patience=4, verbose=0, factor=0.5)
 callbacks = [early_stopping, reduce_lr]
if verbose != -1:
callbacks.append(TqdmCallback(verbose=verbose))
model.compile(optimizer='adam', loss='mse')
 model.fit(
 X_train,
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=0,
validation_data=vali_data,
callbacks=callbacks 
)
 return model<normalization>
data.groupby(['Sex', 'Pclass'])['Survived'].mean()
def load_cases(df, feat):
cases = pd.DataFrame(df[feat].values, index=df['Date'])
 return cases
def scale_fit(data):
scaler = MinMaxScaler()
scaler.fit(scale_reshape(data))
return scaler
def scale_transform(scaler, data):
return scaler.transform(scale_reshape(data)).reshape(data.shape)
 def scale_reshape(data):
results = np.array(data ).reshape(-1, 2)
 return results
class diff_node:
def __init__(self, first=None, v=None, level=0):
self.first = first
self.v = v
self.level = level
self.prev = None
self.next = None
def difference(X, degree):
diff = X
if degree == 0:
root = diff_node(v=diff)
 else:
root = diff_node(first=diff.iloc[0].values.reshape(-1,2))
for i in range(degree):
diff = diff.diff() [1:].reset_index(drop=True)
 if i + 1 >= degree:
root.next = diff_node(v=diff, level=i+1)
 else:
root.next = diff_node(first=diff.iloc[0].values.reshape(-1,2), level=i+1)
 root.next.prev = root
root = root.next
return root
def inverse_difference(node):
results = node.v
if node.level <= 1:
results = np.where(results >= 0.0, results, 0.0)
 while node.prev is not None:
prev = node.prev
first = prev.first
tmp = np.cumsum(results, axis=0)+ first
results = np.concatenate(( first, tmp), axis=0)
 if prev.level <= 1:
results = np.where(results >= 0.0, results, 0.0)
 node = prev
return np.array(results )<train_model>
train_1=train.loc[(train.Sex=='male')]
test_1=test.loc[(test.Sex=='male')]
train_2=train.loc[(( train.Pclass<=2)&(train.Sex=='female')) ]
test_2=test.loc[(( test.Pclass<=2)&(test.Sex=='female')) ]
train_3=train.loc[(( train.Pclass>2)&(train.Sex=='female')) ]
test_3=test.loc[(( test.Pclass>2)&(test.Sex=='female')) ]
class covid19_forecaster:
def __init__(self, feat, Country, Province, random_state=0):
self.scalar = None
self.model = None
self.model_input_shape = None
self.history = None
self.last_timestep = None
self.train_date = None
self.difference_node = None
self.feat = feat
self.Country = Country
self.Province = Province
self.random_state = random_state
def fit(self, df, time_steps=10, diff_degrees=0, vali_size=0.2, batch_size=1, epochs=1, verbose=1 
):
train_stepped = self._prepare_data(df, time_steps, diff_degrees)
 train_stepped_scaled = scale_transform(self.scalar, train_stepped)
 train, vali = train_test_split(train_stepped_scaled, test_size=vali_size, random_state=self.random_state)
 self.model_input_shape = -1, time_steps, len(self.feat)
 self.model = fit_lstm(train, vali,self.model_input_shape, batch_size, epochs, verbose)
 ')
 self.last_timestep = train_stepped_scaled[-1, :]
def forecast(self, forecast_steps):
predictions_scaled = []
X_test, y_test = self.last_timestep[:-1], self.last_timestep[-1]
for i in range(forecast_steps):
X_test = slide1_window(X_test, y_test)
 y_test = self.model.predict(X_test.reshape(self.model_input_shape)) [0, :] * 1.03
predictions_scaled.append(y_test)
 predictions_scaled = np.array(predictions_scaled)
 predictions = self._prediction_postprocess(self.scalar, predictions_scaled, self.difference_node)
 start_date = pd.to_datetime(self.train_date[-1], format='%Y-%m-%d')
 pred_dates = self._generate_dates(start_date, forecast_steps)
 return pd.DataFrame(predictions, index=pred_dates, columns=self.feat)
 def _prepare_data(self, df, time_steps, diff_degrees):
cases = load_cases(df, self.feat)
 self.history = cases
self.difference_node = difference(cases, diff_degrees)
 difference_value = self.difference_node.v
self.scalar = scale_fit(difference_value)
 train_stepped = timesteps(difference_value, time_steps)
 self.train_date = cases.index[time_steps:]
return train_stepped
def _prediction_postprocess(self, scaler, predictions, node, append=True):
preds = scaler.inverse_transform(predictions.reshape(-1, len(self.feat)))
 preds = preds.reshape(-1, len(self.feat))
tmp_node = deepcopy(node)
 if append:
tmp_node.v = tmp_node.v.append(pd.DataFrame(preds), ignore_index=True)
 preds = inverse_difference(tmp_node)[-len(predictions):]
else:
tmp_node.v = preds
preds = inverse_difference(tmp_node)
 return np.array(preds)
 Province_State: %s'%(self.Country, self.Province))
def _generate_dates(self, start_date, length):
dates = []
for i in range(length):
new_date = start_date + timedelta(days=int(i + 1))
dates.append(new_date.strftime('%Y-%m-%d'))
return dates
def _indextodatetime(self, *datas):
results = []
for data in datas:
dates = data.index.to_series().apply(lambda x: pd.to_datetime(x, format='%Y-%m-%d')).values
values = data.values
new_data = pd.Series(values, index=dates)
 results.append(new_data)
 return tuple(results )<load_from_csv>
col_final=['Age_T', 'Fare_T', 'Alone', 'Family_Size', 'FamSurvived', 'Title_Miss', 'Title_Mr', 'Title_Mrs']
def forecast(train_path, test_path):
df_train = load_csv(train_path)
 df_test = load_csv(test_path)
 cp = []
for i, row in df_test.iterrows() :
v = row['Country_Region'], row['Province_State']
if v not in cp:
cp.append(v)
 feats = ['ConfirmedCases', 'Fatalities']
train_start = 40
train_end = 84
time_steps = 10
diff_degrees = 1
vali_size = 0.15
batch_size = 8
epochs = 200
verbose = -1
forecast_steps = 30
predictions = []
for Country, Province in tqdm(cp):
df = df_train[(df_train['Country_Region']==Country)&(df_train['Province_State']==Province)]
train = df[train_start:train_end]
random_state = np.random.randint(0, 20, 1)[0]
forecaster = covid19_forecaster(feats, Country, Province, random_state=random_state)
 forecaster.fit(train, time_steps=time_steps, diff_degrees=diff_degrees, vali_size=vali_size,
batch_size=batch_size, epochs=epochs, verbose=verbose)
 preds = forecaster.forecast(forecast_steps)
 preds = preds.values.tolist()
for v in train[-13:][feats].values.tolist() :
predictions.append(v)
 for p in preds:
predictions.append(p)
 return predictions<predict_on_test>
param_lgb ={'n_estimators': [100, 500, 1000, 2000],
'max_depth':[1,2,3,4,5],
'num_leaves': [2,4,6,8,10],
'min_child_samples ': [2,5,10,15,20],
'min_child_weight': [1e-5, 1e-3, 1e-2, 1e-1, 1, 1e1, 1e2, 1e3, 1e4],
'subsample':[0.4,0.5,0.6,0.7,0.8,0.9,1],
'colsample_bytree':[0.4,0.5,0.6,0.7,0.8,0.9,1],
'reg_alpha': [0, 1e-1, 1, 2, 5, 7, 10, 50, 100],
'reg_lambda': [0, 1e-1, 1, 5, 10, 20, 50, 100]}
param_log={'C': [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, 3, 5, 7, 10, 15, 20, 25, 30, 50],
'penalty':['l1', 'l2', 'elasticnet', 'none'],
'class_weight': [1, 3, 10],
'max_iter': [10000]}
param_rf= {'n_estimators': [10 ,25,50,100,125,250,500,1000],
'max_features': ['auto', 'sqrt'],
'max_depth': [1,2,3,4,5,10],
'min_samples_split': [2, 5, 10, 15, 25],
'min_samples_leaf': [1, 2, 5, 10]}
paramsStack= {'final_estimator__multi_class':['ovr'],
'final_estimator__max_iter':[10000],
'final_estimator__solver':['saga'],
'final_estimator__penalty':['l1', 'l2', 'elasticnet', 'none'],
'final_estimator__l1_ratio':[0.5],
'final_estimator__C':np.linspace(start = 0.0001, stop = 10, num = 100)}
searchParamsModels={'n_iter':50,'scoring':'accuracy','cv':10,'refit':True,'random_state':RS,'verbose':0,'n_jobs':-1}
clf_lgb=lgb.LGBMClassifier(random_state=RS, silent=True, metric='None')
 clf_log=LogisticRegression(random_state = RS, solver='saga', l1_ratio=0.5)
 clf_rf=RandomForestClassifier(random_state = RS)
 stackM=LogisticRegression(random_state=RS)
 l_param=[param_log,param_rf,param_lgb]
l_clf=[clf_log,clf_rf,clf_lgb]
predictions = forecast(train_path, test_path )<save_to_csv>
def tune_stack_predict(X_train,y_train,X_test,listModels,listSearchParamM,stackModel,searchParamsModels,listSearchParamS):
models_local=[]
X_test_c=X_test.copy()
for clf, param in zip(listModels,listSearchParamM):
rs_clf=RandomizedSearchCV(estimator=clf, param_distributions=param, **searchParamsModels)
 rs_clf.fit(X_train, y_train)
 rs_clf_name=type(clf ).__name__[0:5]
y_pred=rs_clf.best_estimator_.predict(X_test)
 X_test_c[rs_clf_name]=np.asarray(y_pred)
 models_local.append([rs_clf_name,rs_clf.best_estimator_])
 stack_clf=StackingClassifier(estimators=models_local, final_estimator=stackModel,cv=10, verbose=0)
 rs_stack=RandomizedSearchCV(estimator=stack_clf, param_distributions=listSearchParamS, **searchParamsModels)
 rs_stack.fit(X_train,y_train)
 rs_stack_name=type(stack_clf ).__name__[0:5]
y_pred=rs_stack.best_estimator_.predict(X_test)
 X_test_c[rs_stack_name]=np.asarray(y_pred)
 models_local.append([rs_stack_name,rs_stack.best_estimator_])
 return X_test_c, models_local
df_sub = pd.read_csv(sub_path)
 feats = ['ConfirmedCases', 'Fatalities']
df_sub[feats] = predictions
df_sub.to_csv('submission.csv', index=False )<set_options>
pd.options.display.max_rows = 500
pd.options.display.max_columns = 500
%matplotlib inline 
<load_from_csv>
param_meta1={'cv': 10,
'estimators': [['Logis',
LogisticRegression(C=0.4, class_weight=10, l1_ratio=0.5, max_iter=10000,
penalty='l1', random_state=81, solver='saga')],
['Rando',
RandomForestClassifier(max_depth=3, min_samples_leaf=5, min_samples_split=5,
n_estimators=25, random_state=81)],
['LGBMC',
LGBMClassifier(colsample_bytree=1, max_depth=3, metric='None',
min_child_samples =10, min_child_weight=10.0, n_estimators=500,
n_jobs=4, num_leaves=6, random_state=81, reg_alpha=2,
reg_lambda=50, subsample=0.6)]],
'final_estimator': LogisticRegression(C=3.3334, l1_ratio=0.5, max_iter=10000, multi_class='ovr',
random_state=81, solver='saga'),
'n_jobs': None,
'passthrough': False,
'stack_method': 'auto',
'verbose': 0}
param_meta3={'cv': 10,
'estimators': [['Logis',
LogisticRegression(C=30, class_weight=1, l1_ratio=0.5, max_iter=10000,
penalty='none', random_state=81, solver='saga')],
['Rando',
RandomForestClassifier(max_depth=10, max_features='sqrt', min_samples_leaf=2,
min_samples_split=15, n_estimators=1000,
random_state=81)],
['LGBMC',
LGBMClassifier(colsample_bytree=0.5, max_depth=1, metric='None',
min_child_samples =15, min_child_weight=0.01, n_estimators=1000,
n_jobs=4, num_leaves=6, random_state=81, reg_alpha=0,
reg_lambda=50, subsample=0.6)]],
'final_estimator': LogisticRegression(C=3.3334, l1_ratio=0.5, max_iter=10000, multi_class='ovr',
random_state=81, solver='saga'),
'n_jobs': None,
'passthrough': False,
'stack_method': 'auto',
'verbose': 0}
train = pd.read_csv('.. /input/covid19-global-forecasting-week-4/train.csv')
 us_before = pd.read_csv('.. /input/jhu-covid19-data-with-us-state-data-prior-to-mar-9/covid19_train_data_us_states_before_march_09_new.csv')
 update =(train['Country_Region'] == 'US')&(train.Date <= '2020-03-09')
 df = train[update]
us_before = us_before[(us_before['Country.Region'] == 'US')&(us_before.Date <= '2020-03-09')]
us_before = us_before[['Province.State', 'Country.Region', 'Date', 'ConfirmedCases', 'Fatalities']]
us_before.columns = ['Province_State', 'Country_Region', 'Date', 'ConfirmedCases_new', 'Fatalities_new']
df.shape, us_before.shape
df = df.merge(us_before, how='left', on=['Province_State', 'Country_Region', 'Date'] ).fillna(0)
 train.loc[update, 'ConfirmedCases'] = df['ConfirmedCases_new'].values
train.loc[update, 'Fatalities'] = df['Fatalities_new'].values
train['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 train['day'] = train.Date.dt.dayofyear
train['geo'] = ['_'.join(x)for x in zip(train['Country_Region'], train['Province_State'])]
train = train[train.Date <= '2020-04-15'].copy()
train<feature_engineering>
stack_clf1=StackingClassifier(**param_meta1)
 stack_clf3=StackingClassifier(**param_meta3 )
test = pd.read_csv('.. /input/covid19-global-forecasting-week-4/test.csv')
 test['Province_State'].fillna('', inplace=True)
 test['Date'] = pd.to_datetime(test['Date'])
 test['day'] = test.Date.dt.dayofyear
test['geo'] = ['_'.join(x)for x in zip(test['Country_Region'], test['Province_State'])]
test
day_min = train['day'].min()
train['day'] -= day_min
test['day'] -= day_min
min_test_val_day = test.day.min()
max_test_val_day = train.day.max()
max_test_day = test.day.max()
num_days = max_test_day + 1
min_test_val_day, max_test_val_day, num_days
train['ForecastId'] = -1
test['Id'] = -1
test['ConfirmedCases'] = 0
test['Fatalities'] = 0
data = pd.concat([train,
test[test.day > max_test_val_day][train.columns]
] ).reset_index(drop=True)
 dates = data[data['geo'] == 'France_'].Date.values
region_meta = pd.read_csv('.. /input/covid19-forecasting-metadata/region_metadata.csv')
 region_meta['Province_State'].fillna('', inplace=True)
 region_meta['geo'] = ['_'.join(x)for x in zip(region_meta['Country_Region'], region_meta['Province_State'],)]
region_meta
set(data.geo.unique())- set(region_meta.geo.unique())
 region_meta = data[['geo']].merge(region_meta, how='left', on='geo')
 region_meta = region_meta.groupby('geo' ).first()
population = np.log1p(region_meta[['population']])
 population = population[['population']].values
population.shape
lockdown_date = pd.read_csv('.. /input/covid19-lockdown-dates-by-country/countryLockdowndates.csv')
 lockdown_date['Province'].fillna('', inplace=True)
 lockdown_date['Date'] = pd.to_datetime(lockdown_date['Date'], dayfirst=True)
 lockdown_date['lock_day'] = lockdown_date.Date.dt.dayofyear
lockdown_date['lock_day'] -= day_min
lockdown_date['geo'] = ['_'.join(x)for x in zip(lockdown_date['Country/Region'], lockdown_date['Province'])]
lockdown_date
lockdown_date = data[['geo', 'day']].merge(lockdown_date[['geo', 'lock_day']], how='left', on=['geo'])
 lockdown_date['locked'] = 1 *(lockdown_date['day'] > lockdown_date['lock_day'])
 lockdown_date = lockdown_date.pivot(index='geo', columns='day', values='locked' ).values
lockdown_date
geo_data = data.pivot(index='geo', columns='day', values='ForecastId')
 num_geo = geo_data.shape[0]
geo_data
geo_id = {}
for i,g in enumerate(geo_data.index):
geo_id[g] = i
num_geo
ConfirmedCases = data.pivot(index='geo', columns='day', values='ConfirmedCases')
 Fatalities = data.pivot(index='geo', columns='day', values='Fatalities')
 cases = np.log1p(ConfirmedCases.values)
 deaths = np.log1p(Fatalities.values)
 def get_dataset(start_pred, num_train, lag_period, continents_ids_base, country_ids_base, cases, deaths,
population, time_cases, time_deaths, lockdown_date):
days = np.arange(start_pred - num_train + 1, start_pred + 1)
 lag_cases = np.vstack([cases[:, d - lag_period : d] for d in days])
 lag_deaths = np.vstack([deaths[:, d - lag_period : d] for d in days])
 target_cases = np.vstack([cases[:, d : d + 1] for d in days])
 target_deaths = np.vstack([deaths[:, d : d + 1] for d in days])
 continents_ids = np.vstack([continents_ids_base for d in days])
 country_ids = np.vstack([country_ids_base for d in days])
 population = np.vstack([population for d in days])
 time_case = np.vstack([time_cases[:, d - 1: d ] for d in days])
 time_death = np.vstack([time_deaths[:, d - 1 : d ] for d in days])
 lockdown = [get_lockdown(lockdown_date, d)for d in days]
lockdown_case = np.vstack([l[0] for l in lockdown])
 lockdown_death = np.vstack([l[1] for l in lockdown])
 start_pred = np.hstack([d * np.ones(( cases.shape[0],)) for d in days] ).astype('int')
 return(lag_cases, lag_deaths, target_cases, target_deaths,
continents_ids, country_ids, population, time_case, time_death,
lockdown_case, lockdown_death, start_pred, days)
 def update_valid_dataset(data, pred_death, pred_case, cases, deaths): 
(lag_cases, lag_deaths, target_cases, target_deaths,
continents_ids, country_ids, population, time_case, time_death,
lockdown_case, lockdown_death, start_pred, days)= data
day = days[-1] + 1
new_lag_cases = np.hstack([lag_cases[:, 1:], pred_case])
 new_lag_deaths = np.hstack([lag_deaths[:, 1:], pred_death])
 new_target_cases = cases[:, day:day+1]
new_target_deaths = deaths[:, day:day+1]
new_continents_ids = continents_ids
new_country_ids = country_ids
new_population = population
new_time_death, new_time_case = update_time(time_death, time_case, pred_death, pred_case)
 new_lockdown_case = lockdown_case
new_lockdown_death = lockdown_death
new_start_pred = 1 + start_pred
new_days = 1 + days
return(new_lag_cases, new_lag_deaths, new_target_cases, new_target_deaths,
new_continents_ids, new_country_ids, new_population,
new_time_case, new_time_death, new_lockdown_case, new_lockdown_death,
new_start_pred, new_days)
 def train_model(train, valid, start_lag_death, end_lag_death, num_lag_case, num_val, score, cases, deaths,):
alpha = 3
lr_death = Ridge(alpha=alpha, fit_intercept=True)
 lr_case = Ridge(alpha=alpha, fit_intercept=False)
(train_death_score, train_case_score, train_pred_death, train_pred_case, 
)= fit_eval(lr_death, lr_case, train, start_lag_death, end_lag_death, num_lag_case, fit=True, score=score,
cases=cases, deaths=deaths)
 death_scores = []
case_scores = []
death_pred = []
case_pred = []
for i in range(num_val): 
(valid_death_score, valid_case_score, valid_pred_death, valid_pred_case, 
)= fit_eval(lr_death, lr_case, valid, start_lag_death, end_lag_death, num_lag_case,
fit=False, score=score,
cases=cases, deaths=deaths)
 death_scores.append(valid_death_score)
 case_scores.append(valid_case_score)
 death_pred.append(valid_pred_death)
 case_pred.append(valid_pred_case)
 if 0:
print('val death: %0.3f' % valid_death_score,
'val case: %0.3f' % valid_case_score,
'val : %0.3f' % np.mean([valid_death_score, valid_case_score]),
flush=True)
 valid = update_valid_dataset(valid, valid_pred_death, valid_pred_case, cases, deaths)
 if score:
death_scores = np.sqrt(np.mean([s**2 for s in death_scores]))
case_scores = np.sqrt(np.mean([s**2 for s in case_scores]))
if 0:
print('train death: %0.3f' % train_death_score,
'train case: %0.3f' % train_case_score,
'val death: %0.3f' % death_scores,
'val case: %0.3f' % case_scores,
'val : %0.3f' %(( death_scores + case_scores)/ 2),
flush=True)
 else:
print('%0.4f' % case_scores,
', %0.4f' % death_scores,
'= %0.4f' %(( death_scores + case_scores)/ 2),
flush=True)
 death_pred = np.hstack(death_pred)
 case_pred = np.hstack(case_pred)
 return death_scores, case_scores, death_pred, case_pred
def get_country_ids(last_train, case_threshold):
countries = [g.split('_')[0] for g in geo_data.index]
countries = pd.factorize(countries)[0]
countries[cases[:, :last_train+1].max(axis=1)< np.log1p(case_threshold)] = -1
countries = pd.factorize(countries)[0]
country_ids_base = countries.reshape(( -1, 1))
ohe = OneHotEncoder(sparse=False)
 country_ids_base = 0.2 * ohe.fit_transform(country_ids_base)
 return country_ids_base
def get_continent_ids() :
continents = region_meta['continent']
continents = pd.factorize(continents)[0]
continents_ids_base = continents.reshape(( -1, 1))
ohe = OneHotEncoder(sparse=False)
 continents_ids_base = ohe.fit_transform(continents_ids_base)
 return continents_ids_base
def val_score(true, pred):
pred = np.log1p(np.round(np.expm1(pred)- 0.2))
return np.sqrt(mean_squared_error(true.ravel() , pred.ravel()))
def val_score(true, pred):
return np.sqrt(mean_squared_error(true.ravel() , pred.ravel()))
def fit_eval(lr_death, lr_case, data, start_lag_death, end_lag_death, num_lag_case, fit, score, cases, deaths): 
(lag_cases, lag_deaths, target_cases, target_deaths,
continents_ids, country_ids, population, time_case, time_death,
lockdown_case, lockdown_death, start_pred, days)= data
idx = np.arange(lag_cases.shape[0])
 X_trend = predict(lag_deaths, num_lag_death_trend, eps_trend, w_trend, window_trend)
 X_death = np.hstack([lag_cases[:, -start_lag_death:-end_lag_death],
lag_deaths[:, -num_lag_death:],
country_ids,
continents_ids,
population,
time_death,
])
 y_death = target_deaths
y_death_prev = lag_deaths[:, -1:]
if fit:
lr_death.fit(X_death, y_death)
 y_pred_death = lr_death.predict(X_death)
 if 0:
w_t = lag_deaths[:, -1:]
w_t =(0 + w_t)/(0 + np.max(w_t))
w_t = w_t**0.7
w_t = 0.6 * w_t *(lag_deaths[:, -num_lag_death_trend:1-num_lag_death_trend] > np.log1p(death_trend_thr))
w_t = 0.4 *(lag_deaths[:, -num_lag_death_trend:1-num_lag_death_trend] > np.log1p(death_trend_thr))
y_pred_death =(( 1 - w_t)* y_pred_death + w_t * X_trend)
 y_pred_death = np.maximum(y_pred_death, y_death_prev)
 X_trend = predict(lag_cases, num_lag_trend, eps_trend, w_trend, window_trend)
 X_case = np.hstack([lag_cases[:, -num_lag_case:],
country_ids,
continents_ids,
population,
time_case,
lockdown_case[idx, start_pred].reshape(( -1, 1)) ,
])
 y_case = target_cases
y_case_prev = lag_cases[:, -1:]
if fit:
lr_case.fit(X_case, y_case)
 y_pred_case = lr_case.predict(X_case)
 w_t = lag_cases[:, -1:]
w_t =(0 + w_t)/(0 + np.max(w_t))
w_t = np.clip(1.1 * w_t**0.7, 0, 1)
 y_pred_case =(( 1 - w_t)* y_pred_case + w_t * X_trend)
 y_pred_case = np.maximum(y_pred_case, y_case_prev)
 if score:
death_score = val_score(y_death, y_pred_death)
 case_score = val_score(y_case, y_pred_case)
 else:
death_score = 0
case_score = 0
return death_score, case_score, y_pred_death, y_pred_case
def update_time(time_death, time_case, pred_death, pred_case):
new_time_death = np.expm1(time_death)+ c_death *(pred_death >= np.log1p(t_death))
new_time_death = 1 *np.log1p(new_time_death)
 new_time_case = np.expm1(time_case)+ c_case *(pred_case >= np.log1p(t_case))
new_time_case = 1 *np.log1p(new_time_case)
 return new_time_death, new_time_case
c_case = 1
t_case = 1000
c_death = 1
t_death = 100
time_cases = c_case *(cases >= np.log1p(t_case))
time_cases = np.cumsum(time_cases, axis=1)
 time_cases = 1 * np.log1p(time_cases)
 time_cases.shape
time_deaths = c_death *(cases >= np.log1p(t_death))
time_deaths = np.cumsum(time_deaths, axis=1)
 time_deaths = 1 *np.log1p(time_deaths)
 time_deaths.shape
def predict(tdata, num_lag, eps, w, window):
num_pred = 1
num_geo = tdata.shape[0]
days = np.arange(num_lag)
 pred = np.zeros(( num_geo, num_pred))
start_pred = tdata.shape[1]
x0 = np.arange(start_pred ).reshape(( -1, 1))
w_delta =(window - 1)/2
x1 = np.arange(start_pred + w_delta, start_pred + 1 + w_delta ).reshape(( -1, 1))
w0 = np.array([w** i for i in range(num_lag)])
 w0 = w0 / w0.mean()
for i in range(num_geo):
y0 = pd.Series(tdata[i, :])
 y = y0.rolling(window=window ).mean().fillna(method='bfill')
 y = y.diff().fillna(0)
 y = np.clip(y.values, 0, 1e6)
 if y.max() == 0:
pred[i, :] = tdata[i, -1]
continue
x = x0
filter_ =(y > 0)
 x = x[y > 0]
y = y[y > 0]
y = y[-num_lag:]
x = x[-num_lag:]
pad = num_lag - len(y)
 if pad > 0:
y = np.hstack(( np.zeros(pad), y))
xmin = x.min()
x = np.vstack(( np.arange(xmin - pad, xmin ).reshape(( -1, 1)) , x))
w = w0[-len(y):]
if len(y)< num_lag:
pred[i, :] = tdata[i, -1]
continue
y = np.log(y + eps)
 lr = Ridge(fit_intercept=True)
 lr.fit(x, y, w)
 if 0 and lr.coef_[0] > 0:
print('up', lr.coef_, lr.intercept_)
 y_pred = lr.predict(x1)
 y_pred = np.clip(y_pred, -10, np.average(y, weights=w))
y_pred = np.exp(y_pred)- eps
y_pred = np.clip(y_pred, 0, 5)
 y_pred = np.cumsum(y_pred)
 y_pred = y_pred + tdata[i, -1]
pred[i, :] = y_pred
return pred
def get_lockdown(lockdown_date, day):
lockdown = lockdown_date.copy()
lockdown[:, day:] = 0
time_since_lockdown = np.cumsum(lockdown, axis=1)
 lockdown_cases = np.log1p(1e1*np.clip(time_since_lockdown - 12, 0, 100))
lockdown_deaths = np.log1p(1e1*np.clip(time_since_lockdown- 14, 0, 100))
return lockdown_cases, lockdown_deaths
start_lag_death, end_lag_death = 15, 5,
num_train = 7
num_lag_death = 6
num_lag_case = 12
lag_period = 15
case_threshold = 30 
(num_lag_trend, eps_trend, w_trend, window_trend)=(num_lag_case, 1e-5, 1.2, 5)
 num_lag_death_trend = 12
death_trend_thr = 1
case_trend_thr = 1
scores = []
for start_val_delta in range(0, -11, -3):
start_val = min_test_val_day + start_val_delta
last_train = start_val - 1
num_val = max_test_val_day - start_val + 1
first_train = last_train + 1 -(num_train)
 keep_cases = cases
keep_deaths = deaths
print(dates[last_train], '%3d %3d' %(start_val, num_val), end=' ')
 country_ids_base = get_country_ids(last_train, case_threshold)
 continents_ids_base = get_continent_ids()
train_data = get_dataset(last_train, num_train, lag_period,
continents_ids_base, country_ids_base, keep_cases, keep_deaths,
population, time_cases, time_deaths, lockdown_date)
 valid_data = get_dataset(start_val, 1, lag_period,
continents_ids_base, country_ids_base, keep_cases, keep_deaths,
population, time_cases, time_deaths, lockdown_date)
 death_scores, case_scores, _, _ = train_model(train_data, valid_data, start_lag_death, end_lag_death,
num_lag_case, num_val, True, keep_cases, keep_deaths, 
)
 scores.append(( death_scores + case_scores)/ 2)
 scores = np.mean(scores)
 print('mean score: %0.4f' % scores )<create_dataframe>
stack_clf1.fit(train_1[col_final],train_1['Survived'])
 stack_clf3.fit(train_3[col_final],train_3['Survived'] )
def get_sub(start_val_delta=0):
start_val = min_test_val_day + start_val_delta
last_train = start_val - 1
num_val = max_test_val_day - start_val + 1
first_train = last_train + 1 -(num_train)
 keep_cases = cases
keep_deaths = deaths
print(dates[last_train], '%3d %3d' %(start_val, num_val), end=' ')
 country_ids_base = get_country_ids(last_train, case_threshold)
 continents_ids_base = get_continent_ids()
train_data = get_dataset(last_train, num_train, lag_period,
continents_ids_base, country_ids_base, keep_cases, keep_deaths,
population, time_cases, time_deaths, lockdown_date)
 valid_data = get_dataset(start_val, 1, lag_period,
continents_ids_base, country_ids_base, keep_cases, keep_deaths,
population, time_cases, time_deaths, lockdown_date)
 _, _, val_death_preds, val_case_preds = train_model(train_data, valid_data, start_lag_death, end_lag_death,
num_lag_case, num_val, True, keep_cases, keep_deaths, 
)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
sub = test[['Date', 'ForecastId', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub.fillna(0)
 sub = sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
return sub
sub = get_sub()<merge>
y_pred_1=stack_clf1.predict(test_1[col_final])
 y_pred_3=stack_clf3.predict(test_3[col_final] )
known_test = train[['geo', 'day', 'ConfirmedCases', 'Fatalities']
].merge(test[['geo', 'day', 'ForecastId']], how='left', on=['geo', 'day'])
 known_test = known_test[['ForecastId', 'ConfirmedCases', 'Fatalities']][known_test.ForecastId.notnull() ].copy()
known_test
unknow_test = test[test.day > max_test_val_day]
unknow_test
def get_final_sub() :
start_val = max_test_val_day + 1
last_train = start_val - 1
num_val = max_test_day - start_val + 1
first_train = last_train + 1 -(num_train)
 keep_cases = cases
keep_deaths = deaths
print(dates[last_train], '%3d %3d' %(start_val, num_val), end=' ')
 country_ids_base = get_country_ids(last_train, case_threshold)
 continents_ids_base = get_continent_ids()
train_data = get_dataset(last_train, num_train, lag_period,
continents_ids_base, country_ids_base, keep_cases, keep_deaths,
population, time_cases, time_deaths, lockdown_date)
 valid_data = get_dataset(start_val, 1, lag_period,
continents_ids_base, country_ids_base, keep_cases, keep_deaths,
population, time_cases, time_deaths, lockdown_date)
 _, _, val_death_preds, val_case_preds = train_model(train_data, valid_data, start_lag_death, end_lag_death,
num_lag_case, num_val, True, keep_cases, keep_deaths, 
)
 pred_deaths = Fatalities.iloc[:, start_val:start_val+num_val].copy()
pred_deaths.iloc[:, :] = np.expm1(val_death_preds)
 pred_deaths = pred_deaths.stack().reset_index()
pred_deaths.columns = ['geo', 'day', 'Fatalities']
pred_deaths
pred_cases = ConfirmedCases.iloc[:, start_val:start_val+num_val].copy()
pred_cases.iloc[:, :] = np.expm1(val_case_preds)
 pred_cases = pred_cases.stack().reset_index()
pred_cases.columns = ['geo', 'day', 'ConfirmedCases']
pred_cases
print(unknow_test.shape, pred_deaths.shape, pred_cases.shape)
 sub = unknow_test[['Date', 'ForecastId', 'geo', 'day']]
sub = sub.merge(pred_cases, how='left', on=['geo', 'day'])
 sub = sub.merge(pred_deaths, how='left', on=['geo', 'day'])
 sub = sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
sub = pd.concat([known_test, sub])
 sub['ForecastId'] = sub['ForecastId'].astype('int')
 return sub
sub = get_final_sub()<save_to_csv>
y_pred_1=pd.DataFrame({'PassengerId':test_1['PassengerId'],'Survived':y_pred_1 }, dtype=int)
 y_pred_2=pd.DataFrame({'PassengerId':test_2['PassengerId'],'Survived':1 }, dtype=int)
 y_pred_3=pd.DataFrame({'PassengerId':test_3['PassengerId'],'Survived':y_pred_3 }, dtype=int )
<load_from_csv><EOS> 
y_sub=pd.concat([y_pred_1,y_pred_2,y_pred_3], axis=0 ).sort_values('PassengerId')
 y_sub.to_csv("sub.csv",index=False )