Kaggle competition: Google Cloud & NCAA® ML Competition 2020-NCAAW
<SOS> metric: logloss Kaggle data source: google-cloud-ncaa-march-madness-2020-division-1-womens-tournament<load_from_csv>
pd.set_option('max_columns', None)
 plt.style.use('fivethirtyeight')
 %matplotlib inline
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore')
 print("Libraries imported!" )
training_data_original = pd.read_csv('/kaggle/input/inputs/train.csv')
 training_data_original['Date'] = pd.to_datetime(training_data_original['Date'])
 training_data = training_data_original[training_data_original['Country_Region'] != 'Diamond Princess']
countries = training_data['Country_Region'].unique()
countries_main = ['China', 'US', 'Australia', 'Canada']
states = training_data['Province_State'].unique()<load_from_csv>
class BaseModel(object):

 def __init__(self, train_df, test_df, target, features, categoricals=[],
n_splits=3, cv_method="KFold", group=None, task="regression",
parameter_tuning=False, scaler=None, verbose=True):
self.train_df = train_df
self.test_df = test_df
self.target = target
self.features = features
self.n_splits = n_splits
self.categoricals = categoricals
self.cv_method = cv_method
self.group = group
self.task = task
self.parameter_tuning = parameter_tuning
self.scaler = scaler
self.cv = self.get_cv()
self.verbose = verbose
self.params = self.get_params()
self.y_pred, self.score, self.model, self.oof, self.y_val, self.fi_df = self.fit()
def train_model(self, train_set, val_set):
raise NotImplementedError
def get_params(self):
raise NotImplementedError
def convert_dataset(self, x_train, y_train, x_val, y_val):
raise NotImplementedError
def convert_x(self, x):
return x
def calc_metric(self, y_true, y_pred):
if self.task == "classification":
return log_loss(y_true, y_pred)
 elif self.task == "regression":
return np.sqrt(mean_squared_error(y_true, y_pred))
def get_cv(self):
if self.cv_method == "KFold":
cv = KFold(n_splits=self.n_splits, shuffle=True, random_state=42)
 return cv.split(self.train_df)
 elif self.cv_method == "StratifiedKFold":
cv = StratifiedKFold(n_splits=self.n_splits, shuffle=True, random_state=42)
 return cv.split(self.train_df, self.train_df[self.target])
 elif self.cv_method == "TimeSeriesSplit":
cv = TimeSeriesSplit(max_train_size=None, n_splits=self.n_splits)
 return cv.split(self.train_df)
 elif self.cv_method == "GroupKFold":
cv = GroupKFold(n_splits=self.n_splits, shuffle=True, random_state=42)
 return cv.split(self.train_df, self.train_df[self.target], self.group)
 elif self.cv_method == "StratifiedGroupKFold":
cv = StratifiedGroupKFold(n_splits=self.n_splits, shuffle=True, random_state=42)
 return cv.split(self.train_df, self.train_df[self.target], self.group)
 def fit(self):
oof_pred = np.zeros(( self.train_df.shape[0],))
y_vals = np.zeros(( self.train_df.shape[0],))
y_pred = np.zeros(( self.test_df.shape[0],))
if self.group is not None:
if self.group in self.features:
self.features.remove(self.group)
 if self.group in self.categoricals:
self.categoricals.remove(self.group)
 fi = np.zeros(( self.n_splits, len(self.features)))
 if self.scaler is not None:
numerical_features = [f for f in self.features if f not in self.categoricals]
self.train_df[numerical_features] = self.train_df[numerical_features].fillna(self.train_df[numerical_features].median())
 self.test_df[numerical_features] = self.test_df[numerical_features].fillna(self.test_df[numerical_features].median())
 self.train_df[self.categoricals] = self.train_df[self.categoricals].fillna(self.train_df[self.categoricals].mode().iloc[0])
 self.test_df[self.categoricals] = self.test_df[self.categoricals].fillna(self.test_df[self.categoricals].mode().iloc[0])
 if self.scaler == "MinMax":
scaler = MinMaxScaler()
elif self.scaler == "Standard":
scaler = StandardScaler()
df = pd.concat([self.train_df[numerical_features], self.test_df[numerical_features]], ignore_index=True)
 scaler.fit(df[numerical_features])
 x_test = self.test_df.copy()
x_test[numerical_features] = scaler.transform(x_test[numerical_features])
 x_test = [np.absolute(x_test[i])for i in self.categoricals] + [x_test[numerical_features]]
else:
x_test = self.test_df[self.features]
for fold,(train_idx, val_idx)in enumerate(self.cv):
x_train, x_val = self.train_df.loc[train_idx, self.features], self.train_df.loc[val_idx, self.features]
y_train, y_val = self.train_df.loc[train_idx, self.target], self.train_df.loc[val_idx, self.target]
if self.scaler is not None:
x_train[numerical_features] = scaler.transform(x_train[numerical_features])
 x_val[numerical_features] = scaler.transform(x_val[numerical_features])
 x_train = [np.absolute(x_train[i])for i in self.categoricals] + [x_train[numerical_features]]
x_val = [np.absolute(x_val[i])for i in self.categoricals] + [x_val[numerical_features]]
train_set, val_set = self.convert_dataset(x_train, y_train, x_val, y_val)
 model, importance = self.train_model(train_set, val_set)
 fi[fold, :] = importance
conv_x_val = self.convert_x(x_val)
 y_vals[val_idx] = y_val
oof_pred[val_idx] = model.predict(conv_x_val ).reshape(oof_pred[val_idx].shape)
 x_test = self.convert_x(x_test)
 y_pred += model.predict(x_test ).reshape(y_pred.shape)/ self.n_splits
print('Partial score of fold {} is: {}'.format(fold, self.calc_metric(y_val, oof_pred[val_idx])))
 fi_df = pd.DataFrame()
for n in np.arange(self.n_splits):
tmp = pd.DataFrame()
tmp["features"] = self.features
tmp["importance"] = fi[n, :]
tmp["fold"] = n
fi_df = pd.concat([fi_df, tmp], ignore_index=True)
 gfi = fi_df[["features", "importance"]].groupby(["features"] ).mean().reset_index()
fi_df = fi_df.merge(gfi, on="features", how="left", suffixes=('', '_mean'))
loss_score = self.calc_metric(self.train_df[self.target], oof_pred)
 if self.verbose:
print('Our oof loss score is: ', loss_score)
 return y_pred, loss_score, model, oof_pred, y_vals, fi_df
def plot_feature_importance(self, rank_range=[1, 50]):
fig, ax = plt.subplots(1, 1, figsize=(10, 20))
sorted_df = self.fi_df.sort_values(by = "importance_mean", ascending=False ).reset_index().iloc[self.n_splits *(rank_range[0]-1): self.n_splits * rank_range[1]]
sns.barplot(data=sorted_df, x ="importance", y ="features", orient='h')
 ax.set_xlabel("feature importance")
 ax.spines['top'].set_visible(False)
 ax.spines['right'].set_visible(False)
 return sorted_df
state_metadata = pd.read_excel('/kaggle/input/external-data/states.xlsx')
 country_metadata = pd.read_excel('/kaggle/input/external-data/countries.xlsx' )<groupby>
class LgbModel(BaseModel):

 def train_model(self, train_set, val_set):
verbosity = 100 if self.verbose else 0
model = lgb.train(self.params, train_set, num_boost_round = 5000, valid_sets=[train_set, val_set], verbose_eval=verbosity)
 fi = model.feature_importance(importance_type="gain")
 return model, fi
def convert_dataset(self, x_train, y_train, x_val, y_val):
train_set = lgb.Dataset(x_train, y_train, categorical_feature=self.categoricals)
 val_set = lgb.Dataset(x_val, y_val, categorical_feature=self.categoricals)
 return train_set, val_set
def get_params(self):
params = {
'num_leaves': 127,
'min_data_in_leaf': 50,
'max_depth': -1,
'learning_rate': 0.005,
"boosting_type": "gbdt",
"bagging_seed": 11,
"verbosity": -1,
'random_state': 42,
}
if self.task == "regression":
params["objective"] = "regression"
params["metric"] = "rmse"
elif self.task == "classification":
params["objective"] = "binary"
params["metric"] = "binary_logloss"
if self.parameter_tuning == True:
def objective(trial):
train_x, test_x, train_y, test_y = train_test_split(self.train_df[self.features],
self.train_df[self.target],
test_size=0.3, random_state=42)
 dtrain = lgb.Dataset(train_x, train_y, categorical_feature=self.categoricals)
 dtest = lgb.Dataset(test_x, test_y, categorical_feature=self.categoricals)
 hyperparams = {'num_leaves': trial.suggest_int('num_leaves', 24, 1024),
'boosting_type': 'gbdt',
'objective': params["objective"],
'metric': params["metric"],
'max_depth': trial.suggest_int('max_depth', 4, 16),
'min_child_weight': trial.suggest_int('min_child_weight', 1, 20),
'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),
'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),
'bagging_freq': trial.suggest_int('bagging_freq', 1, 7),
'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),
'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),
'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),
'early_stopping_rounds': 100
}
model = lgb.train(hyperparams, dtrain, valid_sets=dtest, verbose_eval=500)
 pred = model.predict(test_x)
 if self.task == "classification":
return log_loss(test_y, pred)
 elif self.task == "regression":
return np.sqrt(mean_squared_error(test_y, pred))
study = optuna.create_study(direction='minimize')
 study.optimize(objective, n_trials=50)
 print('Number of finished trials: {}'.format(len(study.trials)))
 print('Best trial:')
 trial = study.best_trial
print(' Value: {}'.format(trial.value))
print(' Params: ')
 for key, value in trial.params.items() :
print(' {}: {}'.format(key, value))
params = trial.params
params["learning_rate"] = 0.001
plot_optimization_history(study)
 return params
def data_preparation(training_data, countries, states, country_metadata, state_metadata, n_days_case,
n_days_fatal, min_num_cases = 2):
training_data_trun = training_data[training_data['ConfirmedCases'] >= min_num_cases]
conf_cases_dict = {}
fatal_dict = {}
for country in countries:
if country not in countries_main:
training_data_trun_loc = training_data_trun[(training_data_trun['Country_Region'] == country)&(pd.isnull(training_data_trun.Province_State)) ]
training_data_trun_loc = training_data_trun_loc.groupby(['Date'] ).sum()
training_data_trun_loc = training_data_trun_loc.sort_values(by = 'Date')
 if len(training_data_trun_loc['ConfirmedCases'].values)>= n_days_case:
conf_cases_dict[country] = training_data_trun_loc['ConfirmedCases'].values[:n_days_case] / country_metadata[country_metadata['Countries'] == country]['Population'].values[0]
if len(training_data_trun_loc['Fatalities'].values)>= n_days_fatal:
fatal_dict[country] = training_data_trun_loc['Fatalities'].values[:n_days_fatal]
for state in states:
training_data_trun_loc = training_data_trun[training_data_trun['Province_State'] == state]
training_data_trun_loc = training_data_trun_loc.groupby(['Date'] ).sum()
training_data_trun_loc = training_data_trun_loc.sort_values(by = 'Date')
 if len(training_data_trun_loc['ConfirmedCases'].values)>= n_days_case:
conf_cases_dict[state] = training_data_trun_loc['ConfirmedCases'].values[:n_days_case] / state_metadata[state_metadata['States'] == state]['Population'].values[0]
if len(training_data_trun_loc['Fatalities'].values)>= n_days_fatal:
fatal_dict[state] = training_data_trun_loc['Fatalities'].values[:n_days_fatal]
return pd.DataFrame(conf_cases_dict), pd.DataFrame(fatal_dict )<train_on_grid>
class CatbModel(BaseModel):

 def train_model(self, train_set, val_set):
verbosity = 100 if self.verbose else 0
if self.task == "regression":
model = CatBoostRegressor(**self.params)
 elif self.task == "classification":
model = CatBoostClassifier(**self.params)
 model.fit(train_set['X'], train_set['y'], eval_set=(val_set['X'], val_set['y']),
verbose=verbosity, cat_features=self.categoricals)
 return model, model.get_feature_importance()
def convert_dataset(self, x_train, y_train, x_val, y_val):
train_set = {'X': x_train, 'y': y_train}
val_set = {'X': x_val, 'y': y_val}
return train_set, val_set
def get_params(self):
params = { 'task_type': "CPU",
'learning_rate': 0.01,
'iterations': 1000,
'random_seed': 42,
'use_best_model': True
}
if self.task == "regression":
params["loss_function"] = "RMSE"
elif self.task == "classification":
params["loss_function"] = "Logloss"
return params
def fts_training(input_df, rank = 3):
matrix = input_df.values
u, s, v = np.linalg.svd(matrix, full_matrices=False)
 scores = np.matmul(u[:, :rank], np.diag(s[:rank]))
pcs = v[:rank, :]
return scores, pcs<create_dataframe>
class Mish(Layer):
def __init__(self, **kwargs):
super(Mish, self ).__init__(**kwargs)
 def build(self, input_shape):
super(Mish, self ).build(input_shape)
 def call(self, x):
return x * K.tanh(K.softplus(x))
def compute_output_shape(self, input_shape):
return input_shape
class LayerNormalization(keras.layers.Layer):
def __init__(self,
center=True,
scale=True,
epsilon=None,
gamma_initializer='ones',
beta_initializer='zeros',
gamma_regularizer=None,
beta_regularizer=None,
gamma_constraint=None,
beta_constraint=None,
**kwargs):

 super(LayerNormalization, self ).__init__(**kwargs)
 self.supports_masking = True
self.center = center
self.scale = scale
if epsilon is None:
epsilon = K.epsilon() * K.epsilon()
self.epsilon = epsilon
self.gamma_initializer = keras.initializers.get(gamma_initializer)
 self.beta_initializer = keras.initializers.get(beta_initializer)
 self.gamma_regularizer = keras.regularizers.get(gamma_regularizer)
 self.beta_regularizer = keras.regularizers.get(beta_regularizer)
 self.gamma_constraint = keras.constraints.get(gamma_constraint)
 self.beta_constraint = keras.constraints.get(beta_constraint)
 self.gamma, self.beta = None, None
def get_config(self):
config = {
'center': self.center,
'scale': self.scale,
'epsilon': self.epsilon,
'gamma_initializer': keras.initializers.serialize(self.gamma_initializer),
'beta_initializer': keras.initializers.serialize(self.beta_initializer),
'gamma_regularizer': keras.regularizers.serialize(self.gamma_regularizer),
'beta_regularizer': keras.regularizers.serialize(self.beta_regularizer),
'gamma_constraint': keras.constraints.serialize(self.gamma_constraint),
'beta_constraint': keras.constraints.serialize(self.beta_constraint),
}
base_config = super(LayerNormalization, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))
def compute_output_shape(self, input_shape):
return input_shape
def compute_mask(self, inputs, input_mask=None):
return input_mask
def build(self, input_shape):
shape = input_shape[-1:]
if self.scale:
self.gamma = self.add_weight(
 shape=shape,
initializer=self.gamma_initializer,
regularizer=self.gamma_regularizer,
constraint=self.gamma_constraint,
name='gamma', 
)
 if self.center:
self.beta = self.add_weight(
 shape=shape,
initializer=self.beta_initializer,
regularizer=self.beta_regularizer,
constraint=self.beta_constraint,
name='beta', 
)
 super(LayerNormalization, self ).build(input_shape)
 def call(self, inputs, training=None):
mean = K.mean(inputs, axis=-1, keepdims=True)
 variance = K.mean(K.square(inputs - mean), axis=-1, keepdims=True)
 std = K.sqrt(variance + self.epsilon)
 outputs =(inputs - mean)/ std
if self.scale:
outputs *= self.gamma
if self.center:
outputs += self.beta
return outputs
def forecast_trajectories(training_data, countries, states, country_metadata,
state_metadata, loc = None, n_days_case = 30, n_days_fatal = 10,
forecast_days = 10, min_num_cases = 2, model_type = 'ARIMA',
components_modeling = True, rank = 3):
kf = KalmanFilter(initial_state_mean = np.zeros(rank ).tolist() , n_dim_obs = rank)
 conf_cases_df, fatal_df = data_preparation(training_data, countries, states,
country_metadata, state_metadata,
n_days_case = n_days_case,
n_days_fatal = n_days_case,
min_num_cases = min_num_cases)
 pred_countries = conf_cases_df.columns.tolist()
pred_countries_fatal = fatal_df.columns.tolist()
conf_cases_exog_df, fatal_exog_df = data_preparation(training_data, countries, states,
country_metadata, state_metadata,
n_days_case = n_days_case + forecast_days,
n_days_fatal = n_days_case + forecast_days,
min_num_cases = min_num_cases)
 scores_exog, pcs_exog = fts_training(conf_cases_exog_df, rank = rank)
 if len(scores_exog)> 0:
scores_exog = kf.em(scores_exog ).smooth(scores_exog)[0]
scores, pcs = fts_training(conf_cases_df, rank = rank)
 forecasted_scores = []
idx = 0
for score in scores.T:
if components_modeling:
exog = scores_exog[:n_days_case, idx] if len(scores_exog)> 0 else None
pred_exog = scores_exog[n_days_case:, idx] if len(scores_exog)> 0 else None
y = score
else:
exog = scores_exog[:n_days_case, :] if len(scores_exog)> 0 else None
pred_exog = scores_exog[n_days_case:, :] if len(scores_exog)> 0 else None
y = conf_cases_df[loc].values
try:
model = arima.ARIMA(endog = y, exog = exog, order =(4, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
try:
model = arima.ARIMA(endog = y, exog = exog, order =(3, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
try:
model = arima.ARIMA(endog = y, exog = exog, order =(2, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
model = arima.ARIMA(endog = y, exog = exog, order =(1, 0, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 if not components_modeling:
pred_traj = model.forecast(steps = forecast_days,
alpha = 0.001,
exog = pred_exog)[0]
break
else:
forecasted_scores.append(model.forecast(steps = forecast_days,
alpha = 0.001,
exog = pred_exog)[0].tolist())
 idx = idx + 1
if components_modeling:
pred_traj = np.matmul(np.array(forecasted_scores ).T, pcs)
 pred_traj_df = pd.DataFrame(pred_traj, columns = pred_countries)
 for loc in pred_countries:
if loc in country_metadata['Countries'].values.tolist() :
pred_traj_df[loc] = country_metadata[country_metadata['Countries']
== loc]['Population'].values[0] * pred_traj_df[loc]
if loc in state_metadata['States'].values.tolist() :
pred_traj_df[loc] = state_metadata[state_metadata['States']
== loc]['Population'].values[0] * pred_traj_df[loc]
else:
pred_traj_df = pd.DataFrame()
if loc in countries:
pred_traj_df[loc] = country_metadata[country_metadata['Countries']
== loc]['Population'].values[0] * pred_traj
elif loc in states:
pred_traj_df[loc] = state_metadata[state_metadata['States']
== loc]['Population'].values[0] * pred_traj
fatal_scores_exog, fatal_pcs_exog = fts_training(fatal_exog_df, rank = rank)
 if len(fatal_pcs_exog)> 0:
fatal_scores_exog = kf.em(fatal_scores_exog ).smooth(fatal_scores_exog)[0]
fatal_scores, fatal_pcs = fts_training(fatal_df, rank = rank)
 forecasted_fatal_scores = []
idx = 0
for fatal_score in fatal_scores.T:
if components_modeling:
exog = fatal_scores_exog[:n_days_fatal, idx] if len(fatal_scores_exog)> 0 else None
pred_exog = fatal_scores_exog[n_days_fatal:, idx] if len(fatal_scores_exog)> 0 else None
y = fatal_score
else:
exog = fatal_scores_exog[:n_days_fatal, :] if len(fatal_scores_exog)> 0 else None
pred_exog = fatal_scores_exog[n_days_fatal:, :] if len(fatal_scores_exog)> 0 else None
y = fatal_df[loc].values
try:
model = arima.ARIMA(endog = y, exog = exog, order =(4, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
try:
model = arima.ARIMA(endog = y, exog = exog, order =(3, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
try:
model = arima.ARIMA(endog = y, exog = exog, order =(2, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
model = arima.ARIMA(endog = y, exog = exog, order =(1, 0, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 if not components_modeling:
fatal_pred_traj = model.forecast(steps = forecast_days,
alpha = 0.001,
exog = pred_exog)[0]
break
else:
forecasted_fatal_scores.append(model.forecast(steps = forecast_days,
alpha = 0.001,
exog = pred_exog)[0].tolist())
 idx = idx + 1
if components_modeling:
fatal_pred_traj = np.matmul(np.array(forecasted_fatal_scores ).T, fatal_pcs)
 fatal_pred_traj_df = pd.DataFrame(fatal_pred_traj, columns = pred_countries_fatal)
 else:
fatal_pred_traj_df = pd.DataFrame()
if loc in countries:
fatal_pred_traj_df[loc] = country_metadata[country_metadata['Countries']
== loc]['Population'].values[0] * fatal_pred_traj
elif loc in states:
fatal_pred_traj_df[loc] = state_metadata[state_metadata['States']
== loc]['Population'].values[0] * fatal_pred_traj
return pred_traj_df, fatal_pred_traj_df 
<predict_on_test>
class NeuralNetworkModel(BaseModel):

 def train_model(self, train_set, val_set):
inputs = []
embeddings = []
embedding_out_dim = self.params['embedding_out_dim']
n_neuron = self.params['hidden_units']
for i in self.categoricals:
input_ = Input(shape=(1,))
embedding = Embedding(int(np.absolute(self.train_df[i] ).max() + 1), embedding_out_dim, input_length=1 )(input_)
 embedding = Reshape(target_shape=(embedding_out_dim,))(embedding)
 inputs.append(input_)
 embeddings.append(embedding)
 input_numeric = Input(shape=(len(self.features)- len(self.categoricals),))
embedding_numeric = Dense(n_neuron )(input_numeric)
 embedding_numeric = Mish()(embedding_numeric)
 inputs.append(input_numeric)
 embeddings.append(embedding_numeric)
 x = Concatenate()(embeddings)
 for i in np.arange(self.params['hidden_layers'] - 1):
x = Dense(n_neuron //(2 *(i+1)) )(x)
 x = Mish()(x)
 x = Dropout(self.params['hidden_dropout'] )(x)
 x = LayerNormalization()(x)
 if self.task == "regression":
out = Dense(1, activation="linear", name = "out" )(x)
 loss = "mse"
elif self.task == "classification":
out = Dense(1, activation='sigmoid', name = 'out' )(x)
 loss = "binary_crossentropy"
model = Model(inputs=inputs, outputs=out)
 model.compile(loss=loss, optimizer=Adam(lr=1e-04, beta_1=0.9, beta_2=0.999, decay=1e-04))
er = EarlyStopping(patience=10, min_delta=1e-4, restore_best_weights=True, monitor='val_loss')
 ReduceLR = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=7, verbose=1, epsilon=1e-4, mode='min')
 model.fit(train_set['X'], train_set['y'], callbacks=[er, ReduceLR],
epochs=self.params['epochs'], batch_size=self.params['batch_size'],
validation_data=[val_set['X'], val_set['y']])
 fi = np.zeros(len(self.features))
return model, fi
def convert_dataset(self, x_train, y_train, x_val, y_val):
train_set = {'X': x_train, 'y': y_train}
val_set = {'X': x_val, 'y': y_val}
return train_set, val_set
def get_params(self):

 params = {
'input_dropout': 0.0,
'hidden_layers': 2,
'hidden_units': 128,
'embedding_out_dim': 4,
'hidden_activation': 'relu',
'hidden_dropout': 0.05,
'batch_norm': 'before_act',
'optimizer': {'type': 'adam', 'lr': 0.001},
'batch_size': 128,
'epochs': 80
}
return params
pred_traj_df, fatal_pred_traj_df = forecast_trajectories(training_data, countries, states, country_metadata, state_metadata,
loc = 'New York', n_days_case = 55, n_days_fatal = 55,
forecast_days = 33, rank = 3, min_num_cases = 2,
components_modeling = True )<filter>
data_dict = {}
for i in glob.glob('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/*'):
name = i.split('/')[-1].split('.')[0]
if name != 'WTeamSpellings':
data_dict[name] = pd.read_csv(i)
 else:
data_dict[name] = pd.read_csv(i, encoding='cp1252' )
def generate_prediction(test_data, training_data, countries_main, countries, states, min_num_cases):
for country in countries:
print(country)
 if country not in countries_main and country not in excl_list:
test_loc_df = test_data[(test_data['Country_Region'] == country)&(pd.isnull(test_data.Province_State)) ].reset_index()
train_loc_df = training_data[(training_data['Country_Region'] == country)&(pd.isnull(training_data.Province_State)) ].reset_index()
test_start = test_loc_df['Date'][0]
test_end = test_loc_df['Date'][len(test_loc_df)- 1]
train_end = train_loc_df['Date'][len(train_loc_df)- 1]
test_loc_df.loc[(( test_loc_df['Date'] >= test_start)) &(test_loc_df['Date'] <= train_end), 'ConfirmedCases'] = train_loc_df[(train_loc_df['Date'] >= test_start)&(train_loc_df['Date'] <= train_end)]['ConfirmedCases'].values
test_loc_df.loc[(( test_loc_df['Date'] >= test_start)) &(test_loc_df['Date'] <= train_end), 'Fatalities'] = train_loc_df[(train_loc_df['Date'] >= test_start)&(train_loc_df['Date'] <= train_end)]['Fatalities'].values
effective_df = train_loc_df[train_loc_df['ConfirmedCases'] >= min_num_cases]
forecast_days = int(( test_end - train_end ).days)
 if len(effective_df)> 0:
effective_train_start = train_loc_df[train_loc_df['ConfirmedCases'] >= min_num_cases].reset_index() ['Date'][0]
n_days_case = int(( train_end - effective_train_start ).days)+ 1
pred_df, fatal_pred_df = forecast_trajectories(training_data, countries, states,
country_metadata, state_metadata, loc = country,
n_days_case = n_days_case, n_days_fatal = n_days_case,
forecast_days = forecast_days, min_num_cases = min_num_cases,
rank = 3)
 test_loc_df.loc[test_loc_df['Date'] > train_end, 'ConfirmedCases'] = np.maximum.accumulate(pred_df[country].values ).astype(int)
 test_data.loc[(test_data['Country_Region'] == country)&(pd.isnull(test_data.Province_State)) , 'ConfirmedCases'] = test_loc_df['ConfirmedCases'].values
test_loc_df.loc[test_loc_df['Date'] > train_end, 'Fatalities'] = np.maximum.accumulate(fatal_pred_df[country].values ).astype(int)
 test_data.loc[(test_data['Country_Region'] == country)&(pd.isnull(test_data.Province_State)) , 'Fatalities'] = test_loc_df['Fatalities'].values
for state in states:
print(state)
 if str(state)not in excl_list:
test_loc_df = test_data[(test_data['Province_State'] == state)].reset_index()
train_loc_df = training_data[(training_data['Province_State'] == state)].reset_index()
test_start = test_loc_df['Date'][0]
test_end = test_loc_df['Date'][len(test_loc_df)- 1]
train_end = train_loc_df['Date'][len(train_loc_df)- 1]
test_loc_df.loc[(( test_loc_df['Date'] >= test_start)) &(test_loc_df['Date'] <= train_end), 'ConfirmedCases'] = train_loc_df[(train_loc_df['Date'] >= test_start)&(train_loc_df['Date'] <= train_end)]['ConfirmedCases'].values
test_loc_df.loc[(( test_loc_df['Date'] >= test_start)) &(test_loc_df['Date'] <= train_end), 'Fatalities'] = train_loc_df[(train_loc_df['Date'] >= test_start)&(train_loc_df['Date'] <= train_end)]['Fatalities'].values
effective_df = train_loc_df[train_loc_df['ConfirmedCases'] >= min_num_cases]
forecast_days = int(( test_end - train_end ).days)
 if len(effective_df)> 0:
effective_train_start = train_loc_df[train_loc_df['ConfirmedCases'] >= min_num_cases].reset_index() ['Date'][0]
n_days_case = int(( train_end - effective_train_start ).days)+ 1
pred_df, fatal_pred_df = forecast_trajectories(training_data, countries, states,
country_metadata, state_metadata, loc = state,
n_days_case = n_days_case, n_days_fatal = n_days_case,
forecast_days = forecast_days, min_num_cases = min_num_cases,
rank = 3)
 test_loc_df.loc[test_loc_df['Date'] > train_end, 'ConfirmedCases'] = np.maximum.accumulate(pred_df[state].values ).astype(int)
 test_data.loc[test_data['Province_State'] == state, 'ConfirmedCases'] = test_loc_df['ConfirmedCases'].values
test_loc_df.loc[test_loc_df['Date'] > train_end, 'Fatalities'] = np.maximum.accumulate(fatal_pred_df[state].values ).astype(int)
 test_data.loc[test_data['Province_State'] == state, 'Fatalities'] = test_loc_df['Fatalities'].values
return test_data 
<load_from_csv>
data_dict['WNCAATourneySeeds']['Seed'] = data_dict['WNCAATourneySeeds']['Seed'].apply(lambda x: int(x[1:3]))
data_dict[fname].head()
test_data = pd.read_csv('/kaggle/input/inputs/test.csv')
 test_data['Date'] = pd.to_datetime(test_data['Date'] )<feature_engineering>
test = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WSampleSubmissionStage1_2020.csv')
 print(test.shape)
 test.head()
test_data['ConfirmedCases'] = None
test_data['Fatalities'] = None<define_variables>
test = test.drop(['Pred'], axis=1)
 test['Season'] = test['ID'].apply(lambda x: int(x.split('_')[0]))
test['WTeamID'] = test['ID'].apply(lambda x: int(x.split('_')[1]))
test['LTeamID'] = test['ID'].apply(lambda x: int(x.split('_')[2]))
test.head()
excl_list = ['Hubei', 'Belize', 'Chad', 'Eritrea', 'Grenada', 'Guinea-Bissau', 'Holy See',
'Laos', 'Libya', 'Mali', 'Mozambique', 'Saint Kitts and Nevis', 'Somalia',
'Syria', 'nan', 'Saint Barthelemy', 'Virgin Islands', 'Montserrat', 'Diamond Princess']<predict_on_test>
gameCities = pd.merge(data_dict['WGameCities'], data_dict['Cities'], how='left', on=['CityID'])
 cols_to_use = gameCities.columns.difference(train.columns ).tolist() + ["Season", "WTeamID", "LTeamID"]
train = train.merge(gameCities[cols_to_use], how="left", on=["Season", "WTeamID", "LTeamID"])
 train.head()
cols_to_use = data_dict["WSeasons"].columns.difference(train.columns ).tolist() + ["Season"]
train = train.merge(data_dict["WSeasons"][cols_to_use], how="left", on=["Season"])
 train.head()
cols_to_use = data_dict["WTeams"].columns.difference(train.columns ).tolist()
train = train.merge(data_dict["WTeams"][cols_to_use], how="left", left_on=["WTeamID"], right_on=["TeamID"])
 train.drop(['TeamID'], axis=1, inplace=True)
 train = train.merge(data_dict["WTeams"][cols_to_use], how="left", left_on=["LTeamID"], right_on=["TeamID"], suffixes=('_W', '_L'))
train.drop(['TeamID'], axis=1, inplace=True)
 print(train.shape)
 train.head()
predictions = generate_prediction(test_data, training_data, countries_main, countries, states, min_num_cases = 2 )<count_unique_values>
cols_to_use = data_dict['WNCAATourneySeeds'].columns.difference(train.columns ).tolist() + ['Season']
train = train.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=["Season","TeamID"]),
how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'])
 train.drop(['TeamID'], axis=1, inplace=True)
 train = train.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=["Season","TeamID"]),
how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], suffixes=('_W', '_L'))
train.drop(['TeamID'], axis=1, inplace=True)
 print(train.shape)
 train.head()
missing_countries = predictions[(predictions.ConfirmedCases.isnull())&(pd.isnull(predictions.Province_State)) ]['Country_Region'].unique().tolist()
missing_states = predictions[(predictions.ConfirmedCases.isnull())&(pd.notnull(predictions.Province_State)) ]['Province_State'].unique().tolist()<split>
cols_to_use = gameCities.columns.difference(test.columns ).tolist() + ["Season", "WTeamID", "LTeamID"]
test = test.merge(gameCities[cols_to_use].drop_duplicates(subset=["Season", "WTeamID", "LTeamID"]),
how="left", on=["Season", "WTeamID", "LTeamID"])
 del gameCities
gc.collect()
test.head()
cols_to_use = data_dict["WSeasons"].columns.difference(test.columns ).tolist() + ["Season"]
test = test.merge(data_dict["WSeasons"][cols_to_use].drop_duplicates(subset=["Season"]),
how="left", on=["Season"])
 test.head()
cols_to_use = data_dict["WTeams"].columns.difference(test.columns ).tolist()
test = test.merge(data_dict["WTeams"][cols_to_use].drop_duplicates(subset=["TeamID"]),
how="left", left_on=["WTeamID"], right_on=["TeamID"])
 test.drop(['TeamID'], axis=1, inplace=True)
 test = test.merge(data_dict["WTeams"][cols_to_use].drop_duplicates(subset=["TeamID"]),
how="left", left_on=["LTeamID"], right_on=["TeamID"], suffixes=('_W', '_L'))
test.drop(['TeamID'], axis=1, inplace=True)
 test.head()
cols_to_use = data_dict['WNCAATourneySeeds'].columns.difference(test.columns ).tolist() + ['Season']
test = test.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=["Season","TeamID"]),
how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'])
 test.drop(['TeamID'], axis=1, inplace=True)
 test = test.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=["Season","TeamID"]),
how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], suffixes=('_W', '_L'))
test.drop(['TeamID'], axis=1, inplace=True)
 print(test.shape)
 test.head()
def fill_excl_pred(predictions, training_data_original, test_data):
for country in missing_countries:
print(country)
 pred_loc_df = predictions[predictions['Country_Region'] == country]
train_loc_df = training_data_original[training_data_original['Country_Region'] == country]
series_comf_cases = train_loc_df['ConfirmedCases'].values
series_fatal = train_loc_df['Fatalities'].values
test_start = test_data[test_data['Country_Region'] == country]['Date'].values[0]
series_comf_cases_test = training_data_original[(training_data_original['Country_Region'] == country)&(training_data_original['Date'] >= test_start)]['ConfirmedCases']
series_fatal_test = training_data_original[(training_data_original['Country_Region'] == country)&(training_data_original['Date'] >= test_start)]['Fatalities']
if len(series_comf_cases)> 0:
regressor = LinearRegression()
regressor.fit(np.arange(len(series_comf_cases_test)).reshape(-1, 1), series_comf_cases_test)
 comf_cases_pred = regressor.predict(np.arange(10, 40 ).reshape(-1, 1))
regressor.fit(np.arange(len(series_fatal_test)).reshape(-1, 1), series_fatal_test)
 fatal_pred = regressor.predict(np.arange(10, 40 ).reshape(-1, 1))
else:
comf_cases_pred = []
fatal_pred = []
conf_cases_loc = np.concatenate(( series_comf_cases_test, comf_cases_pred), axis=0)
 fatal_loc = np.concatenate(( series_fatal_test, fatal_pred), axis=0)
 predictions.loc[predictions['Country_Region'] == country, 'ConfirmedCases'] = conf_cases_loc.astype(int)
 predictions.loc[predictions['Country_Region'] == country, 'Fatalities'] = fatal_loc.astype(int)
 for state in missing_states:
print(state)
 pred_loc_df = predictions[predictions['Province_State'] == state]
train_loc_df = training_data_original[training_data_original['Province_State'] == state]
series_comf_cases = train_loc_df['ConfirmedCases'].values
series_fatal = train_loc_df['Fatalities'].values
test_start = test_data[test_data['Province_State'] == state]['Date'].values[0]
series_comf_cases_test = training_data_original[(training_data_original['Province_State'] == state)&(training_data_original['Date'] >= test_start)]['ConfirmedCases']
series_fatal_test = training_data[(training_data_original['Province_State'] == state)&(training_data_original['Date'] >= test_start)]['Fatalities']
regressor = LinearRegression()
regressor.fit(np.arange(len(series_comf_cases_test)).reshape(-1, 1), series_comf_cases_test)
 comf_cases_pred = regressor.predict(np.arange(10, 40 ).reshape(-1, 1))
regressor.fit(np.arange(len(series_fatal_test)).reshape(-1, 1), series_fatal_test)
 fatal_pred = regressor.predict(np.arange(10, 40 ).reshape(-1, 1))
conf_cases_loc = np.concatenate(( series_comf_cases_test, comf_cases_pred), axis=0)
 fatal_loc = np.concatenate(( series_fatal_test, fatal_pred), axis=0)
 predictions.loc[predictions['Province_State'] == state, 'ConfirmedCases'] = conf_cases_loc.astype(int)
 predictions.loc[predictions['Province_State'] == state, 'Fatalities'] = fatal_loc.astype(int)
 return predictions 
<predict_on_test>
not_exist_in_test = [c for c in train.columns.values.tolist() if c not in test.columns.values.tolist() ]
print(not_exist_in_test)
 train = train.drop(not_exist_in_test, axis=1)
 train.head()
prediction_final = fill_excl_pred(predictions, training_data_original, test_data )<data_type_conversions>
team_win_score = regularSeason.groupby(['Season', 'WTeamID'] ).agg({'WScore':['sum', 'count', 'var']} ).reset_index()
team_win_score.columns = [' '.join(col ).strip() for col in team_win_score.columns.values]
team_loss_score = regularSeason.groupby(['Season', 'LTeamID'] ).agg({'LScore':['sum', 'count', 'var']} ).reset_index()
team_loss_score.columns = [' '.join(col ).strip() for col in team_loss_score.columns.values]
del regularSeason
gc.collect()
predictions_csv = pd.DataFrame()
predictions_csv['ForecastId'] = prediction_final['ForecastId'].astype(int)
 predictions_csv['ConfirmedCases'] = prediction_final['ConfirmedCases'].astype(float)
 predictions_csv['Fatalities'] = prediction_final['Fatalities'].astype(float )<save_to_csv>
train = pd.merge(train, team_win_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'WTeamID'])
 train = pd.merge(train, team_loss_score, how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'LTeamID'])
 train = pd.merge(train, team_loss_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'LTeamID'])
 train = pd.merge(train, team_win_score, how='left', left_on=['Season', 'LTeamID_x'], right_on=['Season', 'WTeamID'])
 train.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)
 train.head()
predictions_csv.to_csv('submission.csv', index = False )<set_options>
test = pd.merge(test, team_win_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'WTeamID'])
 test = pd.merge(test, team_loss_score, how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'LTeamID'])
 test = pd.merge(test, team_loss_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'LTeamID'])
 test = pd.merge(test, team_win_score, how='left', left_on=['Season', 'LTeamID_x'], right_on=['Season', 'WTeamID'])
 test.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)
 test.head()
pd.set_option('max_rows', 1000)
<define_variables>
def preprocess(df):
df['x_score'] = df['WScore sum_x'] + df['LScore sum_y']
df['y_score'] = df['WScore sum_y'] + df['LScore sum_x']
df['x_count'] = df['WScore count_x'] + df['LScore count_y']
df['y_count'] = df['WScore count_y'] + df['WScore count_x']
df['x_var'] = df['WScore var_x'] + df['LScore count_y']
df['y_var'] = df['WScore var_y'] + df['WScore var_x']
return df
train = preprocess(train)
 test = preprocess(test )
WINDOW_X_SIZE = 60
WINDOW_Y_SIZE = 1
PUBLIC_LEADERBOARD = False
DRIVE = False
KAGGLE = True
FILE_NAME = 'submission.csv'
if DRIVE:
path = '/content/drive/My Drive/PROYECTOS/COVID-19/'
elif KAGGLE:
path = '.. /input/covid19-global-forecasting-week-2/'
else:
path = './'
if KAGGLE:
path_input = path
path_output = '.. /working/'
path_population = '.. /input/locations-population/'
else:
path_input = path + 'input/'
path_output = path + 'output/'
path_population = path_input 
<load_from_csv>
train_win = train.copy()
train_los = train.copy()
train_win = train_win[['Seed_W', 'Seed_L', 'TeamName_W', 'TeamName_L',
'x_score', 'y_score', 'x_count', 'y_count', 'x_var', 'y_var']]
train_los = train_los[['Seed_L', 'Seed_W', 'TeamName_L', 'TeamName_W',
'y_score', 'x_score', 'x_count', 'y_count', 'x_var', 'y_var']]
train_win.columns = ['Seed_1', 'Seed_2', 'TeamName_1', 'TeamName_2',
'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']
train_los.columns = ['Seed_1', 'Seed_2', 'TeamName_1', 'TeamName_2',
'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']
test = test[['ID', 'Seed_W', 'Seed_L', 'TeamName_W', 'TeamName_L',
'x_score', 'y_score', 'x_count', 'y_count', 'x_var', 'y_var']]
test.columns = ['ID', 'Seed_1', 'Seed_2', 'TeamName_1', 'TeamName_2',
'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']
df_data = pd.read_csv(path_input + 'train.csv', sep=',')
 df_test = pd.read_csv(path_input + 'test.csv', sep=',')
 df_sample_sub = pd.read_csv(path_input + 'submission.csv', sep=',')
 df_population = pd.read_csv(path_population + 'locations_population.csv', sep=',')
<define_variables>
def feature_engineering(df):
df['Seed_diff'] = df['Seed_1'] - df['Seed_2']
df['Score_diff'] = df['Score_1'] - df['Score_2']
df['Count_diff'] = df['Count_1'] - df['Count_2']
df['Var_diff'] = df['Var_1'] - df['Var_2']
df['Mean_score1'] = df['Score_1'] / df['Count_1']
df['Mean_score2'] = df['Score_2'] / df['Count_2']
df['Mean_score_diff'] = df['Mean_score1'] - df['Mean_score2']
df['FanoFactor_1'] = df['Var_1'] / df['Mean_score1']
df['FanoFactor_2'] = df['Var_2'] / df['Mean_score2']
return df
train_win = feature_engineering(train_win)
 train_los = feature_engineering(train_los)
 test = feature_engineering(test )
date_format = "%Y-%m-%d"
print(f'Train Dates: {df_data.Date.min() } - {df_data.Date.max() } // QT_DAYS: {(datetime.strptime(df_data.Date.max() , date_format)- datetime.strptime(df_data.Date.min() , date_format)).days} ')
 print(f'Test Dates: {df_test.Date.min() } - {df_test.Date.max() } // QT_DAYS: {(datetime.strptime(df_test.Date.max() , date_format)- datetime.strptime(df_test.Date.min() , date_format)).days} ')
 print(f'Window days to be predicted: {(datetime.strptime(df_test.Date.max() , date_format)- datetime.strptime(df_data.Date.max() , date_format)).days}')
 window_test_days =(datetime.strptime(df_test.Date.max() , date_format)- datetime.strptime(df_data.Date.max() , date_format)).days 
<feature_engineering>
data = pd.concat(( train_win, train_los)).reset_index(drop=True)
 print(data.shape)
 data.head()
def parseProvinces(df_data, df_test, df_population):
df_data['Province_State'] = df_data['Province_State'].fillna('
df_test['Province_State'] = df_test['Province_State'].fillna('
df_population['Province.State'] = df_population['Province.State'].fillna('
return df_data, df_test, df_population
def isNaN(num):
return num != num
def getDaysElapsedSinceDeltaCases(df, feature_names=['qt_day_100cases'], deltas=[100]):
for feature_name in feature_names:
df[feature_name] = 0
for i, row in tqdm(df.iterrows()):
date = row['Date']
country = row['Country_Region']
province = row['Province_State']
for pos, feature_name in enumerate(feature_names):
date_delta = df_data['Date'][(df_data['Country_Region']==country)&(df_data['Province_State']==province)&(df_data['Date']<=date)&(df_data['ConfirmedCases']>=deltas[pos])].min()
if isNaN(date_delta):
value = 0
else:
value =(datetime.strptime(date, date_format)- datetime.strptime(date_delta, date_format)).days + 1
if value < 0:
value = 0
df[feature_name][i] = value
return df
def normalizeDataModel(vector, feature, axis, dict_normalization, mode='train'):
uni_data_mean = vector.mean(axis=axis)
 uni_data_std = vector.std(axis=axis)
 if mode=='train':
dict_normalization[feature] = {'Mean': uni_data_mean, 'Std': uni_data_std}
return dict_normalization,(vector - uni_data_mean)/uni_data_std
def createCountryDicts(df_data):
unique_countries = df_data['Country_Region'].unique()
dict_countries = {}
dict_countries_inv = {}
for i, country in enumerate(unique_countries):
dict_countries[country] = i
dict_countries_inv[i] = country
return dict_countries, dict_countries_inv
def createProvinceDicts(df_data):
unique_provinces = df_data['Province_State'].unique()
dict_provinces = {}
dict_provinces_inv = {}
for i, province in enumerate(unique_provinces):
dict_provinces[province] = i
dict_provinces_inv[i] = province
return dict_provinces, dict_provinces_inv
def getCountryRepresentation(df, scale=True):
dict_latitudes = {'France': [46.887, 2.552]}
vector = np.zeros(( len(dict_countries.keys()), 5))
for i, country in enumerate(dict_countries):
population = df['Population'][df['Country_Region']==country].values[0]
if df['co_province'][df['Country_Region']==country].unique().shape[0] == 1:
last_confirmed, last_fatalities = df[['ConfirmedCases', 'Fatalities']][df['Country_Region']==country].values[-1]
std_last_confirmed, std_last_fatalities = df[['ConfirmedCases', 'Fatalities']][df['Country_Region']==country].values[-5:].std(axis=0)
 else:
last_confirmed, last_fatalities = df[['Date', 'ConfirmedCases', 'Fatalities']][df['Country_Region']==country].groupby(['Date'] ).sum().values[-1]
std_last_confirmed, std_last_fatalities = df[['Date', 'ConfirmedCases', 'Fatalities']][df['Country_Region']==country].groupby(['Date'] ).sum().values[-5:].std(axis=0)
 vector[i] = np.array([population, std_last_confirmed, std_last_fatalities, last_confirmed, last_fatalities])
 if scale:
scaler = StandardScaler()
vector = scaler.fit_transform(vector)
 return vector
def getProvinceRepresentation(df, scale=True):
vector = np.zeros(( len(dict_provinces.keys()), 13))
countries_raw = getCountryRepresentation(df, scale=False)
 for i, province in enumerate(dict_provinces):
if province == '
lat_, long_ = 0, 0
last_confirmed, last_fatalities = countries_raw[:, -2:].mean(axis=0)
 std_last_confirmed, std_last_fatalities = countries_raw[:, -2:].mean(axis=0)
 qt_1, qt_100, qt_1_000 = df_data[['qt_days_since_1_case', 'qt_days_since_100_cases', 'qt_days_since_1000_cases']]\
[(df_data['Date']==df_data.Date.max())&(df_data['Province_State']=='
population = df_data['Population'][(df_data['Date']==df_data.Date.max())&(df_data['Province_State']=='
vec_country = countries_raw.mean(axis=0)
 else:
last_confirmed, last_fatalities = df[['ConfirmedCases', 'Fatalities']][df['Province_State']==province].values[-1]
std_last_confirmed, std_last_fatalities = df[['ConfirmedCases', 'Fatalities']][df['Province_State']==province].values[-5:].std(axis=0)
 country = df['Country_Region'][df['Province_State']==province].values[0]
qt_1, qt_100, qt_1_000 = df_data[['qt_days_since_1_case', 'qt_days_since_100_cases', 'qt_days_since_1000_cases']][df_data['Province_State']==province].values[-1:].squeeze()
population = df_data['Population'][df_data['Province_State']==province].values[0]
vec_country = countries_raw[dict_countries[country]]
vector[i] = np.hstack([np.array([population, std_last_confirmed, std_last_fatalities, last_confirmed, last_fatalities, qt_1, qt_100, qt_1_000]), vec_country])
 if scale:
scaler = StandardScaler()
vector = scaler.fit_transform(vector)
 return vector
def uniVariateData(df, feature, window_x_size, window_y_size, train=True, return_all_series=False):
num_series = df['Country_Province'].unique().shape[0]
X = np.empty(( num_series, window_x_size, 1))
y = np.empty(( num_series, window_x_size, window_y_size))
if return_all_series:
all_data = X = np.empty(( num_series, window_x_size + window_y_size))
unique_series = df_data['Country_Province'].unique()
if return_all_series:
for i, serie in tqdm(enumerate(unique_series)) :
all_data[i] = df_data[feature][df_data['Country_Province']==serie].values[-(WINDOW_X_SIZE+WINDOW_Y_SIZE):]
return all_data
else:
for i, serie in tqdm(enumerate(unique_series)) :
data = df_data[feature][df_data['Country_Province']==serie].values
if train:
X[i] = data[:window_x_size].reshape(-1, 1)
 else:
X[i] = data[-window_x_size:].reshape(-1, 1)
 if train:
for step_ahead in range(1, window_y_size + 1):
y[i, :, step_ahead - 1] = data[step_ahead:step_ahead + window_x_size]
if train:
return X, y
else:
return X
def create_time_steps(length):
return list(range(-length, 0))
def baseline(history):
return np.mean(history[-1:])
 def show_plot(plot_data, delta, title):
labels = ['History', 'True Future', 'Model Prediction']
marker = ['.-', 'rx', 'go']
time_steps = create_time_steps(plot_data[0].shape[0])
 if delta:
future = delta
else:
future = 0
plt.title(title)
 for i, x in enumerate(plot_data):
if i:
plt.plot(future, plot_data[i], marker[i], markersize=10,
label=labels[i])
 else:
plt.plot(time_steps, plot_data[i].flatten() , marker[i], label=labels[i])
 plt.legend()
plt.xlim([time_steps[0],(future+5)*2])
 plt.xlabel('Time-Step')
 plt.show()
def multi_step_plot(history, true_future, prediction):
plt.figure(figsize=(12, 6))
num_in = create_time_steps(len(history))
num_out = true_future.shape[0]
plt.plot(num_in, np.array(history), label='History')
 plt.plot(np.arange(num_out)/1, np.array(true_future), 'bo',
label='True Future')
 if prediction.any() :
plt.plot(np.arange(num_out)/1, np.array(prediction), 'ro',
label='Predicted Future')
 plt.legend(loc='upper left')
 plt.show()
def testStepPlot(history, prediction):
plt.figure(figsize=(12, 6))
num_in = create_time_steps(len(history))
num_out = prediction.shape[0]
plt.plot(num_in, np.array(history), label='History')
 plt.plot(np.arange(num_out)/1, np.array(prediction), 'ro',
label='Predicted Future')
 plt.legend(loc='upper left')
 plt.show()
def lastTimeStepMse(y_true, y_pred):
return mean_squared_error(y_true[: , -1], y_pred[:, -1])
 def moovingAverage(array, size_window, weights=None):
if weights:
assert size_window==len(weights)
 new_array = np.empty(array.shape[0])
 for i in range(size_window, array.shape[0]):
if weights:
new_array[i] = np.sum(array[i-size_window:i] * np.array(weights))
else:
new_array[i] = np.mean(array[i-size_window:i])
 return new_array
def buildModel(lr=0.001, summary=False):
input_country = Input(shape=[1], name='country')
 input_province = Input(shape=[1], name='province')
 input_confirmed_cases = Input(shape=[WINDOW_X_SIZE, 1], name='in_confirmedcases')
 input_fatalities = Input(shape=[WINDOW_X_SIZE, 1], name='in_fatalities')
 input_trend_confirmed_cases = Input(shape=[WINDOW_X_SIZE-1, 1], name='in_trend_confirmedcases')
 input_trend_fatalities = Input(shape=[WINDOW_X_SIZE-1, 1], name='in_trend_fatalities')
 input_delta_confirmed_cases = Input(shape=[WINDOW_X_SIZE-1, 1], name='in_delta_confirmedcases')
 input_delta_fatalities = Input(shape=[WINDOW_X_SIZE-1, 1], name='in_delta_fatalities')
 country_index = country_representation.shape[0]
province_index = province_representation.shape[0]
embedding_country = Embedding(country_index,
country_representation.shape[1],
weights=[country_representation],
trainable=False )(input_country)
 embedding_province = Embedding(province_index,
province_representation.shape[1],
weights=[province_representation],
trainable=False )(input_province)
 embeddings = concatenate([Flatten()(embedding_country), Flatten()(embedding_province)])
 lstm_confirmed_cases = LSTM(64, return_sequences=True, dropout=0.2, recurrent_dropout=0.2 )(input_confirmed_cases)
 lstm_trend_confirmed_cases = LSTM(64, return_sequences=True, dropout=0.2, recurrent_dropout=0.2 )(input_trend_confirmed_cases)
 lstm_fatalities = LSTM(64, return_sequences=True, dropout=0.2, recurrent_dropout=0.2 )(input_fatalities)
 lstm_trend_fatalities = LSTM(64, return_sequences=True, dropout=0.2, recurrent_dropout=0.2 )(input_trend_fatalities)
 lstm_confirmed_cases = LSTM(64, return_sequences=False, dropout=0.2, recurrent_dropout=0.2 )(lstm_confirmed_cases)
 lstm_trend_confirmed_cases = LSTM(64, return_sequences=False, dropout=0.2, recurrent_dropout=0.2 )(lstm_trend_confirmed_cases)
 lstm_fatalities = LSTM(64, return_sequences=False, dropout=0.2, recurrent_dropout=0.2 )(lstm_fatalities)
 lstm_trend_fatalities = LSTM(64, return_sequences=False, dropout=0.2, recurrent_dropout=0.2 )(lstm_trend_fatalities)
 conf_cases = concatenate([lstm_confirmed_cases, lstm_trend_confirmed_cases, lstm_trend_fatalities, embeddings])
 conf_cases = Dropout(0.4 )(conf_cases)
 conf_cases = Dense(128, activation='selu' )(conf_cases)
 conf_cases = Dropout(0.3 )(conf_cases)
 conf_cases = Dense(64, activation='selu' )(conf_cases)
 conf_cases = Dropout(0.2 )(conf_cases)
 fatalities = concatenate([lstm_fatalities, lstm_trend_fatalities, lstm_trend_confirmed_cases, embeddings])
 fatalities = Dropout(0.4 )(fatalities)
 fatalities = Dense(128, activation='selu' )(fatalities)
 fatalities = Dropout(0.3 )(fatalities)
 fatalities = Dense(64, activation='selu' )(fatalities)
 fatalities = Dropout(0.2 )(fatalities)
 output_confirmed_cases = Dense(WINDOW_Y_SIZE, activation='relu', name='confirmed_cases' )(conf_cases)
 output_fatalities = Dense(WINDOW_Y_SIZE, activation='relu', name='fatalities' )(fatalities)
 model = Model(inputs=[input_country, input_province, input_confirmed_cases, input_fatalities, input_trend_confirmed_cases, input_trend_fatalities,
input_delta_confirmed_cases, input_delta_fatalities],
outputs=[output_confirmed_cases, output_fatalities])
 model.compile(loss_weights=[1, 1], loss='mse', optimizer=Adam(learning_rate=lr, clipvalue=1.0), metrics=[lastTimeStepMse, 'mape'])
 if summary:
print(model.summary())
 return model 
<data_type_conversions>
categoricals = ["TeamName_1", "TeamName_2"]
for c in categoricals:
le = LabelEncoder()
data[c] = data[c].fillna("NaN")
 data[c] = le.fit_transform(data[c])
 test[c] = le.transform(test[c])
 data.head()
if PUBLIC_LEADERBOARD:
df_data = df_data[df_data['Date']<='2020-03-18']
df_data['dt_datetime'] = pd.to_datetime(df_data['Date'])
 df_data['co_weekday'] = df_data.dt_datetime.dt.weekday
df_test['dt_datetime'] = pd.to_datetime(df_test['Date'])
 df_test['co_weekday'] = df_test.dt_datetime.dt.weekday
df_data, df_test, df_population = parseProvinces(df_data, df_test, df_population)
 df_data = df_data.merge(df_population[['Province.State', 'Country.Region', 'Population']], how='left', left_on=['Province_State', 'Country_Region'], right_on=['Province.State', 'Country.Region'])
 df_test = df_test.merge(df_population[['Province.State', 'Country.Region', 'Population']], how='left', left_on=['Province_State', 'Country_Region'], right_on=['Province.State', 'Country.Region'])
 df_data['Population'][df_data['Population'].isna() ] = df_data['Population'].median()
df_test['Population'][df_test['Population'].isna() ] = df_test['Population'].median()
df_data = getDaysElapsedSinceDeltaCases(df_data, ['qt_days_since_1_case', 'qt_days_since_100_cases', 'qt_days_since_1000_cases'], deltas=[1, 100, 1_000])
 df_data['Country_Province'] = df_data.apply(lambda x: x['Province_State'] + '_' + x['Country_Region'], axis=1)
 df_data_original = df_data.copy()
df_test_original = df_test.copy()
dict_countries, dict_countries_inv = createCountryDicts(df_data)
 dict_provinces, dict_provinces_inv = createProvinceDicts(df_data)
 df_data['co_country'] = df_data['Country_Region'].apply(lambda x: dict_countries[x])
 df_test['co_country'] = df_test['Country_Region'].apply(lambda x: dict_countries[x])
 df_data['co_province'] = df_data['Province_State'].apply(lambda x: dict_provinces[x])
 df_test['co_province'] = df_test['Province_State'].apply(lambda x: dict_provinces[x])
 country_representation = getCountryRepresentation(df_data)
 province_representation = getProvinceRepresentation(df_data)
 df_data['ConfirmedCases'] = np.log1p(df_data['ConfirmedCases'])
 df_data['Fatalities'] = np.log1p(df_data['Fatalities'])
 data_confirmed_cases = uniVariateData(df_data, 'ConfirmedCases', WINDOW_X_SIZE, WINDOW_Y_SIZE, return_all_series=True)
 data_fatalities = uniVariateData(df_data, 'Fatalities', WINDOW_X_SIZE, WINDOW_Y_SIZE, return_all_series=True)
 X_confirmedcases = data_confirmed_cases[:, :WINDOW_X_SIZE]
y_confirmedcases = data_confirmed_cases[:, WINDOW_X_SIZE:]
X_fatalities = data_fatalities[:, :WINDOW_X_SIZE]
y_fatalities = data_fatalities[:, WINDOW_X_SIZE:]
X_countries = uniVariateData(df_data, 'co_country', WINDOW_X_SIZE, WINDOW_Y_SIZE, train=False)
 X_province = uniVariateData(df_data, 'co_province', WINDOW_X_SIZE, WINDOW_Y_SIZE, train=False)
 X_countries = np.expand_dims(X_countries[:, 0, 0], axis=1)
 X_province = np.expand_dims(X_province[:, 0, 0], axis=1)
 X_trend_confirmed_cases = np.expm1(X_confirmedcases[:, 1:])- np.expm1(X_confirmedcases[:, :-1])
 X_trend_fatalities = np.expm1(X_fatalities[:, 1:])- np.expm1(X_fatalities[:, :-1])
 for i in range(X_trend_confirmed_cases.shape[0]):
X_trend_confirmed_cases[i] = moovingAverage(X_trend_confirmed_cases[i], size_window=7)
 X_trend_fatalities[i] = moovingAverage(X_trend_fatalities[i], size_window=7)
 X_delta_confirmed_cases = np.nan_to_num(( np.expm1(X_confirmedcases[:, 1:])- np.expm1(X_confirmedcases[:, :-1])) /np.expm1(X_confirmedcases[:, :-1]), nan=0, posinf=0)
 X_delta_fatalities = np.nan_to_num(( np.expm1(X_fatalities[:, 1:])- np.expm1(X_fatalities[:, :-1])) /np.expm1(X_fatalities[:, :-1]), nan=0, posinf=0)
 mean_series = X_confirmedcases.mean(axis=1)
 series_with_no_confimred_cases = {i for i, serie in enumerate(mean_series)if serie==0}
series_with_confimred_cases = [i for i in range(X_confirmedcases.shape[0])if i not in series_with_no_confimred_cases]
X_confirmedcases = np.expand_dims(X_confirmedcases, axis=2)
 X_fatalities = np.expand_dims(X_fatalities, axis=2)
 X_trend_confirmed_cases = np.expand_dims(X_trend_confirmed_cases, axis=2)
 X_trend_fatalities = np.expand_dims(X_trend_fatalities, axis=2)
 X_delta_confirmed_cases = np.expand_dims(X_delta_confirmed_cases, axis=2)
 X_delta_fatalities = np.expand_dims(X_delta_fatalities, axis=2)
 print(X_confirmedcases.mean() , y_confirmedcases.mean() , X_confirmedcases.std() , y_confirmedcases.std())
 print(X_fatalities.mean() , y_fatalities.mean() , X_fatalities.std() , y_fatalities.std())
 x_train = [X_countries[series_with_confimred_cases], X_province[series_with_confimred_cases], X_confirmedcases[series_with_confimred_cases],
X_fatalities[series_with_confimred_cases], X_trend_confirmed_cases[series_with_confimred_cases], X_trend_fatalities[series_with_confimred_cases],
X_delta_confirmed_cases[series_with_confimred_cases], X_delta_fatalities[series_with_confimred_cases]]
y_train = [y_confirmedcases[series_with_confimred_cases], y_fatalities[series_with_confimred_cases]] 
<define_variables>
target = 'result'
features = data.columns.values.tolist()
features.remove(target )
dt_ini = datetime.strptime(df_data_original.Date.max() , date_format)- timedelta(WINDOW_X_SIZE)
 dt_end = datetime.strptime(df_data_original.Date.max() , date_format)
 dt_max_test = df_test['Date'].max()
X_test_confirmedcases = X_confirmedcases
X_test_fatalities = X_fatalities
X_test_countries = X_countries
X_test_province = X_province
X_test_trend_confirmed_cases = X_trend_confirmed_cases
X_test_trend_fatalities = X_trend_fatalities
X_test_delta_confirmed_cases = X_delta_confirmed_cases
X_test_delta_fatalities = X_delta_fatalities
i = 0
X_final_confirmedcases = X_test_confirmedcases
X_final_fatalities = X_test_fatalities
while dt_end.strftime(date_format)< dt_max_test:
print('==='*20)
 print(f'dt_ini: {dt_ini.strftime(date_format)}')
 print(f'dt_end: {dt_end.strftime(date_format)}')
 print('==='*20)
 x_test = [X_test_countries, X_test_province, X_test_confirmedcases, X_test_fatalities, X_test_trend_confirmed_cases, X_test_trend_fatalities,
X_delta_confirmed_cases, X_delta_fatalities]
y_test_predictions = model.predict(x_test)
 y_pred_confirmedcases_unscaled = y_test_predictions[0]
y_pred_fatalities_unscaled = y_test_predictions[1]
y_pred_confirmedcases_unscaled[list(series_with_no_confimred_cases)] = 0
y_pred_fatalities_unscaled[list(series_with_no_confimred_cases)] = 0
if i==0:
X_final_confirmedcases = np.concatenate([X_final_confirmedcases.squeeze() , y_pred_confirmedcases_unscaled], axis=1)
 X_final_fatalities = np.concatenate([X_final_fatalities.squeeze() , y_pred_fatalities_unscaled], axis=1)
 else:
X_final_confirmedcases = np.concatenate([X_final_confirmedcases, y_pred_confirmedcases_unscaled], axis=1)
 X_final_fatalities = np.concatenate([X_final_fatalities, y_pred_fatalities_unscaled], axis=1)
 X_test_confirmedcases = np.expand_dims(X_final_confirmedcases[: ,-WINDOW_X_SIZE:], axis=2)
 X_test_fatalities = np.expand_dims(X_final_fatalities[:, -WINDOW_X_SIZE:], axis=2)
 X_test_trend_confirmed_cases = np.expm1(X_test_confirmedcases[:, 1:])- np.expm1(X_test_confirmedcases[:, :-1])
 X_test_trend_fatalities = np.expm1(X_test_fatalities[:, 1:])- np.expm1(X_test_fatalities[:, :-1])
 for i in range(X_trend_confirmed_cases.shape[0]):
X_test_trend_confirmed_cases[i] = np.expand_dims(moovingAverage(X_test_trend_confirmed_cases[i], size_window=5), axis=1)
 X_test_trend_fatalities[i] = np.expand_dims(moovingAverage(X_test_trend_fatalities[i], size_window=5), axis=1)
 X_test_delta_confirmed_cases = np.nan_to_num(( np.expm1(X_test_confirmedcases[:, 1:])- np.expm1(X_test_confirmedcases[:, :-1])) /np.expm1(X_test_confirmedcases[:, :-1]), nan=0, posinf=0)
 X_test_delta_fatalities = np.nan_to_num(( np.expm1(X_test_fatalities[:, 1:])- np.expm1(X_test_fatalities[:, :-1])) /np.expm1(X_test_fatalities[:, :-1]), nan=0, posinf=0)
 dt_ini += timedelta(WINDOW_Y_SIZE)
 dt_end += timedelta(WINDOW_Y_SIZE)
 i += 1 
<merge>
nn = NeuralNetworkModel(data, test, target, features, categoricals=categoricals, n_splits=10,
cv_method="StratifiedKFold", group=None, task="classification", scaler="MinMax", verbose=True )
df_test = pd.merge(df_test_original, df_data[['Province_State', 'Country_Region', 'Date', 'ConfirmedCases', 'Fatalities']], how='left', on=['Province_State', 'Country_Region', 'Date'])
 df_test['ConfirmedCases'][df_test['Date']>df_data['Date'].max() ] = X_final_confirmedcases[:, WINDOW_X_SIZE:].flatten()
df_test['Fatalities'][df_test['Date']>df_data['Date'].max() ] = X_final_fatalities[:, WINDOW_X_SIZE:].flatten()
submission = df_test[['ForecastId', 'ConfirmedCases', 'Fatalities']][df_test.Date>=df_test.Date.min() ].reset_index(drop = True)
 if not PUBLIC_LEADERBOARD:
assert submission.shape[0] == df_sample_sub.shape[0]
assert submission.shape[1] == df_sample_sub.shape[1]
submission['ConfirmedCases'] = np.expm1(submission['ConfirmedCases'])
 submission['Fatalities'] = np.expm1(submission['Fatalities'])
 print(submission.describe())
 submission.to_csv(path_output + FILE_NAME, sep=',', index=False, header=True)
<compute_test_metric>
lgbm = LgbModel(data, test, target, features, categoricals=categoricals, n_splits=10,
cv_method="StratifiedKFold", group=None, task="classification", scaler=None, verbose=True )
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)) )<load_from_csv>
catb = CatbModel(data, test, target, features, categoricals=categoricals, n_splits=10,
cv_method="StratifiedKFold", group=None, task="classification", scaler=None, verbose=True )
pd.set_option('mode.chained_assignment', None)
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'] )<feature_engineering>
submission_df = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WSampleSubmissionStage1_2020.csv')
 submission_df['Pred'] = 0.7 * lgbm.y_pred + 0.2 * catb.y_pred + 0.1 * nn.y_pred
submission_df
<compute_test_metric><EOS> 
submission_df.to_csv('submission.csv', index=False )