Kaggle competition: Google Cloud & NCAA® ML Competition 2020-NCAAW
param_grid = {'n_estimators': [1000]}
def gridSearchCV(model, X_Train, y_Train, param_grid, cv=10, scoring='neg_mean_squared_error'):
start = time.time()
grid_cv = GridSearchCV(model, param_grid, cv=10, scoring="neg_mean_squared_error")
 grid_cv.fit(X_Train, y_Train)
 return grid_cv.best_estimator_<categorify>
pd.set_option('max_columns', None)
 plt.style.use('fivethirtyeight')
 %matplotlib inline
py.init_notebook_mode(connected=True)
 warnings.filterwarnings('ignore')
 print("Libraries imported!" )
le = preprocessing.LabelEncoder()<data_type_conversions>
class BaseModel(object):

 def __init__(self, train_df, test_df, target, features, categoricals=[],
n_splits=3, cv_method="KFold", group=None, task="regression",
parameter_tuning=False, scaler=None, verbose=True):
self.train_df = train_df
self.test_df = test_df
self.target = target
self.features = features
self.n_splits = n_splits
self.categoricals = categoricals
self.cv_method = cv_method
self.group = group
self.task = task
self.parameter_tuning = parameter_tuning
self.scaler = scaler
self.cv = self.get_cv()
self.verbose = verbose
self.params = self.get_params()
self.y_pred, self.score, self.model, self.oof, self.y_val, self.fi_df = self.fit()
def train_model(self, train_set, val_set):
raise NotImplementedError
def get_params(self):
raise NotImplementedError
def convert_dataset(self, x_train, y_train, x_val, y_val):
raise NotImplementedError
def convert_x(self, x):
return x
def calc_metric(self, y_true, y_pred):
if self.task == "classification":
return log_loss(y_true, y_pred)
 elif self.task == "regression":
return np.sqrt(mean_squared_error(y_true, y_pred))
def get_cv(self):
if self.cv_method == "KFold":
cv = KFold(n_splits=self.n_splits, shuffle=True, random_state=42)
 return cv.split(self.train_df)
 elif self.cv_method == "StratifiedKFold":
cv = StratifiedKFold(n_splits=self.n_splits, shuffle=True, random_state=42)
 return cv.split(self.train_df, self.train_df[self.target])
 elif self.cv_method == "TimeSeriesSplit":
cv = TimeSeriesSplit(max_train_size=None, n_splits=self.n_splits)
 return cv.split(self.train_df)
 elif self.cv_method == "GroupKFold":
cv = GroupKFold(n_splits=self.n_splits, shuffle=True, random_state=42)
 return cv.split(self.train_df, self.train_df[self.target], self.group)
 elif self.cv_method == "StratifiedGroupKFold":
cv = StratifiedGroupKFold(n_splits=self.n_splits, shuffle=True, random_state=42)
 return cv.split(self.train_df, self.train_df[self.target], self.group)
 def fit(self):
oof_pred = np.zeros(( self.train_df.shape[0],))
y_vals = np.zeros(( self.train_df.shape[0],))
y_pred = np.zeros(( self.test_df.shape[0],))
if self.group is not None:
if self.group in self.features:
self.features.remove(self.group)
 if self.group in self.categoricals:
self.categoricals.remove(self.group)
 fi = np.zeros(( self.n_splits, len(self.features)))
 if self.scaler is not None:
numerical_features = [f for f in self.features if f not in self.categoricals]
self.train_df[numerical_features] = self.train_df[numerical_features].fillna(self.train_df[numerical_features].median())
 self.test_df[numerical_features] = self.test_df[numerical_features].fillna(self.test_df[numerical_features].median())
 self.train_df[self.categoricals] = self.train_df[self.categoricals].fillna(self.train_df[self.categoricals].mode().iloc[0])
 self.test_df[self.categoricals] = self.test_df[self.categoricals].fillna(self.test_df[self.categoricals].mode().iloc[0])
 if self.scaler == "MinMax":
scaler = MinMaxScaler()
elif self.scaler == "Standard":
scaler = StandardScaler()
df = pd.concat([self.train_df[numerical_features], self.test_df[numerical_features]], ignore_index=True)
 scaler.fit(df[numerical_features])
 x_test = self.test_df.copy()
x_test[numerical_features] = scaler.transform(x_test[numerical_features])
 x_test = [np.absolute(x_test[i])for i in self.categoricals] + [x_test[numerical_features]]
else:
x_test = self.test_df[self.features]
for fold,(train_idx, val_idx)in enumerate(self.cv):
x_train, x_val = self.train_df.loc[train_idx, self.features], self.train_df.loc[val_idx, self.features]
y_train, y_val = self.train_df.loc[train_idx, self.target], self.train_df.loc[val_idx, self.target]
if self.scaler is not None:
x_train[numerical_features] = scaler.transform(x_train[numerical_features])
 x_val[numerical_features] = scaler.transform(x_val[numerical_features])
 x_train = [np.absolute(x_train[i])for i in self.categoricals] + [x_train[numerical_features]]
x_val = [np.absolute(x_val[i])for i in self.categoricals] + [x_val[numerical_features]]
train_set, val_set = self.convert_dataset(x_train, y_train, x_val, y_val)
 model, importance = self.train_model(train_set, val_set)
 fi[fold, :] = importance
conv_x_val = self.convert_x(x_val)
 y_vals[val_idx] = y_val
oof_pred[val_idx] = model.predict(conv_x_val ).reshape(oof_pred[val_idx].shape)
 x_test = self.convert_x(x_test)
 y_pred += model.predict(x_test ).reshape(y_pred.shape)/ self.n_splits
print('Partial score of fold {} is: {}'.format(fold, self.calc_metric(y_val, oof_pred[val_idx])))
 fi_df = pd.DataFrame()
for n in np.arange(self.n_splits):
tmp = pd.DataFrame()
tmp["features"] = self.features
tmp["importance"] = fi[n, :]
tmp["fold"] = n
fi_df = pd.concat([fi_df, tmp], ignore_index=True)
 gfi = fi_df[["features", "importance"]].groupby(["features"] ).mean().reset_index()
fi_df = fi_df.merge(gfi, on="features", how="left", suffixes=('', '_mean'))
loss_score = self.calc_metric(self.train_df[self.target], oof_pred)
 if self.verbose:
print('Our oof loss score is: ', loss_score)
 return y_pred, loss_score, model, oof_pred, y_vals, fi_df
def plot_feature_importance(self, rank_range=[1, 50]):
fig, ax = plt.subplots(1, 1, figsize=(10, 20))
sorted_df = self.fi_df.sort_values(by = "importance_mean", ascending=False ).reset_index().iloc[self.n_splits *(rank_range[0]-1): self.n_splits * rank_range[1]]
sns.barplot(data=sorted_df, x ="importance", y ="features", orient='h')
 ax.set_xlabel("feature importance")
 ax.spines['top'].set_visible(False)
 ax.spines['right'].set_visible(False)
 return sorted_df
df_out.ForecastId = df_out.ForecastId.astype('int')
 df_out.tail()<save_to_csv>
class LgbModel(BaseModel):

 def train_model(self, train_set, val_set):
verbosity = 100 if self.verbose else 0
model = lgb.train(self.params, train_set, num_boost_round = 5000, valid_sets=[train_set, val_set], verbose_eval=verbosity)
 fi = model.feature_importance(importance_type="gain")
 return model, fi
def convert_dataset(self, x_train, y_train, x_val, y_val):
train_set = lgb.Dataset(x_train, y_train, categorical_feature=self.categoricals)
 val_set = lgb.Dataset(x_val, y_val, categorical_feature=self.categoricals)
 return train_set, val_set
def get_params(self):
params = {
'num_leaves': 127,
'min_data_in_leaf': 50,
'max_depth': -1,
'learning_rate': 0.005,
"boosting_type": "gbdt",
"bagging_seed": 11,
"verbosity": -1,
'random_state': 42,
}
if self.task == "regression":
params["objective"] = "regression"
params["metric"] = "rmse"
elif self.task == "classification":
params["objective"] = "binary"
params["metric"] = "binary_logloss"
if self.parameter_tuning == True:
def objective(trial):
train_x, test_x, train_y, test_y = train_test_split(self.train_df[self.features],
self.train_df[self.target],
test_size=0.3, random_state=42)
 dtrain = lgb.Dataset(train_x, train_y, categorical_feature=self.categoricals)
 dtest = lgb.Dataset(test_x, test_y, categorical_feature=self.categoricals)
 hyperparams = {'num_leaves': trial.suggest_int('num_leaves', 24, 1024),
'boosting_type': 'gbdt',
'objective': params["objective"],
'metric': params["metric"],
'max_depth': trial.suggest_int('max_depth', 4, 16),
'min_child_weight': trial.suggest_int('min_child_weight', 1, 20),
'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),
'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),
'bagging_freq': trial.suggest_int('bagging_freq', 1, 7),
'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),
'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),
'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),
'early_stopping_rounds': 100
}
model = lgb.train(hyperparams, dtrain, valid_sets=dtest, verbose_eval=500)
 pred = model.predict(test_x)
 if self.task == "classification":
return log_loss(test_y, pred)
 elif self.task == "regression":
return np.sqrt(mean_squared_error(test_y, pred))
study = optuna.create_study(direction='minimize')
 study.optimize(objective, n_trials=50)
 print('Number of finished trials: {}'.format(len(study.trials)))
 print('Best trial:')
 trial = study.best_trial
print(' Value: {}'.format(trial.value))
print(' Params: ')
 for key, value in trial.params.items() :
print(' {}: {}'.format(key, value))
params = trial.params
params["learning_rate"] = 0.001
plot_optimization_history(study)
 return params
df_out.to_csv('submission.csv', index=False )<load_from_csv>
data_dict = {}
for i in glob.glob('/kaggle/input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WDataFiles_Stage1/*'):
name = i.split('/')[-1].split('.')[0]
if name != 'WTeamSpellings':
data_dict[name] = pd.read_csv(i)
 else:
data_dict[name] = pd.read_csv(i, encoding='cp1252' )
mother_given_path = "/kaggle/input/covid19-global-forecasting-week-2"
train_given_path = os.path.join(mother_given_path, "train.csv")
 test_given_path = os.path.join(mother_given_path, "test.csv")
 train = pd.read_csv(train_given_path)
 test = pd.read_csv(test_given_path)
 train.columns.values[2] = 'Country'
train.columns.values[1] = 'Subplace'
train.fillna("", inplace = True)
 test.columns.values[2] = 'Country'
test.columns.values[1] = 'Subplace'
test.fillna("", inplace = True)
 Countries = train.Country.unique()

 mother_path = "/kaggle/input/covid19week2"
loc2pop_path = os.path.join(mother_path, "loc2popv2.pickle")
 with open(loc2pop_path, 'rb')as f:
loc2pop = pickle.load(f)
 def make_data() :
train.reset_index(drop = True, inplace = True)
 train['Location'] = train[['Subplace', 'Country']].apply(lambda x: '_'.join(x), axis=1)
 for i in range(len(train)) :
if i%100 == 0:
print(i/len(train))
if train.loc[i, 'Subplace'] == "":
train.loc[i, 'Location'] = train.loc[i, 'Country']
elif train.loc[i, 'Subplace'] == train.loc[i, 'Country']:
train.loc[i, 'Location'] = train.loc[i, 'Country']
else:
train.loc[i, 'Location'] = '_'.join([train.loc[i, 'Country'], train.loc[i, 'Subplace']])
 train.loc[i, 'Population'] = loc2pop[train.loc[i, 'Location']]
test.reset_index(drop = True, inplace = True)
 test['Location'] = test[['Subplace', 'Country']].apply(lambda x: '_'.join(x), axis=1)
 for i in range(len(test)) :
if i%100 == 0:
print(i/len(test))
if test.loc[i, 'Subplace'] == "":
test.loc[i, 'Location'] = test.loc[i, 'Country']
elif test.loc[i, 'Subplace'] == test.loc[i, 'Country']:
test.loc[i, 'Location'] = test.loc[i, 'Country']
else:
test.loc[i, 'Location'] = '_'.join([test.loc[i, 'Country'], test.loc[i, 'Subplace']])
 test.loc[i, 'Population'] = loc2pop[test.loc[i, 'Location']]
with open('train_dfv2.csv', 'wb')as f:
pickle.dump(train, f)
 with open('test_dfv2.csv', 'wb')as f:
pickle.dump(test, f)

mother_path = "/kaggle/input/covid19week2"
traindf2_path = os.path.join(mother_path, "train_dfv2.csv")
 testdf2_path = os.path.join(mother_path, "test_dfv2.csv")
 with open(traindf2_path, 'rb')as f:
train = pickle.load(f)
 with open(testdf2_path, 'rb')as f:
test = pickle.load(f)
 Locations = train.Location.unique()

 dic = {}
for loc in Locations:
indexNames = train[train.Location == loc].index
dic[loc] = np.array(train.loc[indexNames, "ConfirmedCases"])
 tmp = np.diff(dic[loc])< 0
if len(np.where(tmp == True)[0])> 0:
print(loc)
 dic = {}
for loc in Locations:
indexNames = train[train.Location == loc].index
dic[loc] = np.array(train.loc[indexNames, "Fatalities"])
 tmp = np.diff(dic[loc])< 0
if len(np.where(tmp == True)[0])> 0:
print(loc)

def make_data() :
train.reset_index(drop = True, inplace = True)
 train['Location'] = train[['Subplace', 'Country']].apply(lambda x: '_'.join(x), axis=1)
 for i in range(len(train)) :
if i%100 == 0:
print(i/len(train))
if train.loc[i, 'Subplace'] == "":
train.loc[i, 'Location'] = train.loc[i, 'Country']
elif train.loc[i, 'Subplace'] == train.loc[i, 'Country']:
train.loc[i, 'Location'] = train.loc[i, 'Country']
else:
train.loc[i, 'Location'] = '_'.join([train.loc[i, 'Country'], train.loc[i, 'Subplace']])
 train.loc[i, 'Population'] = loc2pop[train.loc[i, 'Location']]
test.reset_index(drop = True, inplace = True)
 test['Location'] = test[['Subplace', 'Country']].apply(lambda x: '_'.join(x), axis=1)
 for i in range(len(test)) :
if i%100 == 0:
print(i/len(test))
if test.loc[i, 'Subplace'] == "":
test.loc[i, 'Location'] = test.loc[i, 'Country']
elif test.loc[i, 'Subplace'] == test.loc[i, 'Country']:
test.loc[i, 'Location'] = test.loc[i, 'Country']
else:
test.loc[i, 'Location'] = '_'.join([test.loc[i, 'Country'], test.loc[i, 'Subplace']])
 test.loc[i, 'Population'] = loc2pop[test.loc[i, 'Location']]
for d in [6, 7, 8, 9]:
train.loc[(train.Location == 'Australia_Northern Territory')&(train.Date == '2020-03-{:02}'.format(d)) , 'ConfirmedCases'] = 1.0
for d in [31]:
train.loc[(train.Location == 'Australia_Queensland')&(train.Date == '2020-01-{:02}'.format(d)) , 'ConfirmedCases'] = 3.0
for d in [2, 3]:
train.loc[(train.Location == 'Australia_Queensland')&(train.Date == '2020-02-{:02}'.format(d)) , 'ConfirmedCases'] = 3.0
for d in [25]:
train.loc[(train.Location == 'Canada_Alberta')&(train.Date == '2020-03-{:02}'.format(d)) , 'ConfirmedCases'] = 433.0
for d in [17]:
train.loc[(train.Location == 'China_Guizhou')&(train.Date == '2020-03-{:02}'.format(d)) , 'ConfirmedCases'] = 146.0
for d in [9, 10, 11, 12, 13, 14, 15]:
train.loc[(train.Location == 'France_Saint Barthelemy')&(train.Date == '2020-03-{:02}'.format(d)) , 'ConfirmedCases'] = 3.0
for d in [17, 18, 19, 20, 21, 22, 23, 24, 25,26]:
train.loc[(train.Location == 'Guyana')&(train.Date == '2020-03-{:02}'.format(d)) , 'ConfirmedCases'] = 5.0
for d in [14]:
train.loc[(train.Location == 'US_Alaska')&(train.Date == '2020-03-{:02}'.format(d)) , 'ConfirmedCases'] = 1.0
for d in [18]:
train.loc[(train.Location == 'US_Nevada')&(train.Date == '2020-03-{:02}'.format(d)) , 'ConfirmedCases'] = 76.0
for d in [20]:
train.loc[(train.Location == 'US_Utah')&(train.Date == '2020-03-{:02}'.format(d)) , 'ConfirmedCases'] = 108.0
for d in [20, 21]:
train.loc[(train.Location == 'US_Virgin Islands')&(train.Date == '2020-03-{:02}'.format(d)) , 'ConfirmedCases'] = 3.0
for d in [29, 30]:
train.loc[(train.Location == 'US_Virgin Islands')&(train.Date == '2020-03-{:02}'.format(d)) , 'ConfirmedCases'] = 22.0
for d in [18]:
train.loc[(train.Location == 'US_Washington')&(train.Date == '2020-03-{:02}'.format(d)) , 'ConfirmedCases'] = 1226.0
for d in [21]:
train.loc[(train.Location == 'Canada_Quebec')&(train.Date == '2020-03-{:02}'.format(d)) , 'Fatalities'] = 4.0
for d in [15]:
train.loc[(train.Location == 'Iceland')&(train.Date == '2020-03-{:02}'.format(d)) , 'Fatalities'] = 0.0
for d in [20]:
train.loc[(train.Location == 'Iceland')&(train.Date == '2020-03-{:02}'.format(d)) , 'Fatalities'] = 1.0
for d in [20]:
train.loc[(train.Location == 'India')&(train.Date == '2020-03-{:02}'.format(d)) , 'Fatalities'] = 4.0
for d in [20]:
train.loc[(train.Location == 'Kazakhstan')&(train.Date == '2020-03-{:02}'.format(d)) , 'Fatalities'] = 0.0
for d in [18]:
train.loc[(train.Location == 'Philippines')&(train.Date == '2020-03-{:02}'.format(d)) , 'Fatalities'] = 14.0
for d in [18, 19, 20, 21]:
train.loc[(train.Location == 'Slovakia')&(train.Date == '2020-03-{:02}'.format(d)) , 'Fatalities'] = 0.0
for d in [24]:
train.loc[(train.Location == 'US_Hawaii')&(train.Date == '2020-03-{:02}'.format(d)) , 'Fatalities'] = 0.0
for d in [26, 27]:
train.loc[(train.Location == 'Serbia')&(train.Date == '2020-03-{:02}'.format(d)) , 'Fatalities'] = 4.0
for d in [30]:
train.loc[(train.Location == 'US_Virginia')&(train.Date == '2020-03-{:02}'.format(d)) , 'Fatalities'] = 25.0
with open('train_dfv2.csv', 'wb')as f:
pickle.dump(train, f)
 with open('test_dfv2.csv', 'wb')as f:
pickle.dump(test, f)

mother_path = "/kaggle/input/covid19week2"
train_path = os.path.join(mother_path, "train_dfv2.csv")
 test_path = os.path.join(mother_path, "test_dfv2.csv")
 loc_path = os.path.join(mother_path, "loc2popv2.pickle")
 with open(train_path, 'rb')as f:
train = pickle.load(f)
 train = train[train.Location != 'Diamond Princess']
with open(test_path, 'rb')as f:
test = pickle.load(f)
 with open(loc_path, 'rb')as f:
loc2pop = pickle.load(f)
 Locations = train.Location.unique()

 tf.random.set_seed(0)
 tf.keras.backend.set_floatx('float32')
 class FC(Model):
def __init__(self, input_dims):
super(FC, self ).__init__()
self.input_dims = input_dims
self.inputs = Input(shape=self.input_dims)
 self.dense1 = Dense(200, activation = 'softplus', name = 'dense1')
 self.subdense1 = Dense(50, activation = 'softplus', name = 'sub_dense1')
 self.subdense2 = Dense(1, activation = 'softplus', name = 'super_class')
 self.subdense3 = Dense(50, activation = 'softplus', name = 'sub_dense3')
 self.dense2 = Dense(50, activation = 'softplus', name = 'dense2')
 self.logit = Dense(1, activation ='softplus', name = 'logit')
 def build(self):
x = self.inputs
f = self.dense1(x)
 s = self.subdense1(f)
 self.lat = self.subdense2(s)
 s = self.subdense3(self.lat)
 f = self.dense2(f)
 x = Concatenate()([f, s])
 logit = self.logit(x)
 return tf.keras.Model(inputs=self.inputs, outputs= [logit, self.lat])
 def loss_fn(X, LOGITS):
y, s = X
y_logit, s_logit = LOGITS
y_loss = tf.keras.losses.MeanSquaredError()(y, y_logit)
 s_loss = tf.keras.losses.MeanSquaredError()(s, s_logit)
 return tf.reduce_mean(y_loss + s_loss)
 period = 21

 dic = {}
for loc in Locations:
indexNames = train[train.Location == loc].index
dic[loc] = np.array(train.loc[indexNames, "ConfirmedCases"])
 trainset = []
testset = []
def allround(n):
if n != 0:
integer, decimal = int(np.log10(n)) , np.log10(n)- int(np.log10(n))
return int(10**(decimal)) * 10**integer
else:
return 0
weight_pop = 10000
def extract_feature(sig, pop):
diff = np.diff(sig)
 diff = np.array([allround(n)for n in diff] ).astype(np.float32)
 diff_ratio = weight_pop*diff/pop
return diff_ratio
def target(sig, pop):
diff = np.diff(sig)
 diff = np.array([allround(n)for n in diff] ).astype(np.float32)
 diff_ratio = weight_pop*diff/pop
return diff_ratio[0]
for i, loc in enumerate(Locations):
if loc in ['Korea, South', 'China_Hubei']:
loc_df = dic[loc].copy()
pop = loc2pop[loc]
pop = allround(pop)
 try:
k = np.where(loc_df != 0)[0][0]
except:
k = len(indexNames)-1
try:
critical_point = np.where(loc_df/pop>1e-07)[0][0]
except:
critical_point = len(indexNames)-1
if k + period < len(indexNames)-1:
while k+period < len(indexNames)-1:
tmp = loc_df[k:k+period+1]
diff = extract_feature(tmp[:-1], pop)
 y_tmp = target(tmp[-2:], pop)
 days = 1/(np.maximum(1, k - critical_point-14))
trainset.append([diff, days, y_tmp])
 k+=1
tmp = loc_df[k:k+period+1]
diff = extract_feature(tmp[:-1], pop)
 y_tmp = target(tmp[-2:], pop)
 days = 1/(np.maximum(1, k - critical_point-14))
testset.append([diff, days, y_tmp])
 train_x = np.stack([trainset[i][0] for i in range(len(trainset)) ], axis = 0 ).astype(np.float32)
 train_s = np.stack([trainset[i][1] for i in range(len(trainset)) ], axis = 0 ).astype(np.float32)
 train_y = np.array([trainset[i][2] for i in range(len(trainset)) ] ).astype(np.float32)
 test_x = np.stack([testset[i][0] for i in range(len(testset)) ], axis = 0 ).astype(np.float32)
 test_s = np.stack([testset[i][1] for i in range(len(testset)) ], axis = 0 ).astype(np.float32)
 test_y = np.array([testset[i][2] for i in range(len(testset)) ] ).astype(np.float32)
 n_train_samples = len(train_x)
 n_test_samples = len(test_x)
 train_ds = tf.data.Dataset.from_tensor_slices(
(train_x, train_y, train_s)).shuffle(n_train_samples ).batch(32)
 test_ds = tf.data.Dataset.from_tensor_slices(( test_x, test_y, test_s)).batch(n_test_samples)
 folder_name = os.path.join(mother_path, "mlp_p_{}".format(period))
try:
os.mkdir(folder_name)
 except OSError:
pass
lr = 0.01
model= FC(train_x.shape[1:] ).build()
compute_loss = loss_fn
optimizer = tf.keras.optimizers.SGD(lr=lr, decay=1e-6, momentum=0.9, nesterov=True)
 train_loss = tf.keras.metrics.Mean(name='train_loss')
 test_loss = tf.keras.metrics.Mean(name='test_loss')
 def train_step(inputs):
X, Y, S = inputs
with tf.GradientTape() as tape:
logits = model(X)
 loss = compute_loss([Y, S], logits)
 gradients = tape.gradient(loss, model.trainable_variables)
 optimizer.apply_gradients(zip(gradients, model.trainable_variables))
train_loss(loss)
 def test_step(inputs):
X, Y, S = inputs
logits = model(X)
 loss = compute_loss([Y, S], logits)
 test_loss(loss)
 def runs(log_freq = 1):
for epoch in range(5000):
for epoch_x, epoch_y, epoch_s in train_ds:
inputs = [epoch_x, epoch_y, epoch_s]
train_step(inputs)
 for epoch_x, epoch_y, epoch_s in test_ds:
test_step([epoch_x, epoch_y, epoch_s])
 template = 'epoch: {}, train_loss: {}, test_loss: {}'
print(template.format(epoch+1,
train_loss.result() ,
test_loss.result()))
tf.keras.backend.clear_session()

 train_con_for_path = os.path.join(mother_path, "train_for_dfv2.csv")
 with open(train_con_for_path, 'rb')as f:
train_for = pickle.load(f)
 dic_for = {}
for loc in Locations:
indexNames = train_for[train_for.Location == loc].index
dic_for[loc] = np.array(train_for.loc[indexNames, "c"])
 dic_fat = {}
for loc in Locations:
indexNames = train[train.Location == loc].index
dic_fat[loc] = np.array(train.loc[indexNames, "Fatalities"])
 trainset = []
testset = []
weight_con = 100
def extract_fat_feature(fat, con):
diff = np.diff(fat)
 ratio = diff/con[:-1]
ratio[np.where(con[:-1] == 0)[0]] = 0
return weight_con*ratio
def target_fat(fat, con):
diff = np.diff(fat)
 ratio = diff/con[:-1]
ratio[np.where(con[:-1] == 0)[0]] = 0
return weight_con*ratio[0]
for i, loc in enumerate(Locations):
if loc in ['Korea, South', 'China_Hubei']:
loc_df = dic[loc].copy()
loc_fat_df = dic_fat[loc].copy()
pop = loc2pop[loc]
pop = allround(pop)
 try:
k = np.where(loc_df != 0)[0][0]
except:
k = len(indexNames)-1
try:
critical_point = np.where(loc_df/pop>1e-07)[0][0]
except:
critical_point = len(indexNames)-1
if k + period < len(indexNames)-1:
while k+period < len(indexNames)-1:
tmp_fat = loc_fat_df[k:k+period+1]
tmp = loc_df[k:k+period+1]
diff = extract_fat_feature(tmp_fat[:-1], tmp[:-1])
 y_tmp = target_fat(tmp_fat[-2:],tmp[-2:])
 days = 1/(np.maximum(1, k - critical_point-14))
trainset.append([diff, days, y_tmp])
 k+=1
tmp_fat = loc_fat_df[k:k+period+1]
tmp = loc_df[k:k+period+1]
diff = extract_fat_feature(tmp_fat[:-1], tmp[:-1])
 y_tmp = target_fat(tmp_fat[-2:],tmp[-2:])
 days = 1/(np.maximum(1, k - critical_point-14))
testset.append([diff, days, y_tmp])
 train_x = np.stack([trainset[i][0] for i in range(len(trainset)) ], axis = 0 ).astype(np.float32)
 train_s = np.stack([trainset[i][1] for i in range(len(trainset)) ], axis = 0 ).astype(np.float32)
 train_y = np.array([trainset[i][2] for i in range(len(trainset)) ] ).astype(np.float32)
 test_x = np.stack([testset[i][0] for i in range(len(testset)) ], axis = 0 ).astype(np.float32)
 test_s = np.stack([testset[i][1] for i in range(len(testset)) ], axis = 0 ).astype(np.float32)
 test_y = np.array([testset[i][2] for i in range(len(testset)) ] ).astype(np.float32)
 n_train_samples = len(train_x)
 n_test_samples = len(test_x)
 train_ds = tf.data.Dataset.from_tensor_slices(
(train_x, train_y, train_s)).shuffle(n_train_samples ).batch(32)
 test_ds = tf.data.Dataset.from_tensor_slices(( test_x, test_y, test_s)).batch(n_test_samples)
 folder_name = os.path.join(mother_path, "mlp_fat_p_{}".format(period))
try:
os.mkdir(folder_name)
 except OSError:
pass
lr = 0.005
model= FC(train_x.shape[1:] ).build()
compute_loss = loss_fn
optimizer = tf.keras.optimizers.SGD(lr=lr, decay=1e-6, momentum=0.9, nesterov=True)
 train_loss = tf.keras.metrics.Mean(name='train_loss')
 test_loss = tf.keras.metrics.Mean(name='test_loss')
 def train_fat_step(inputs):
X, Y, S = inputs
with tf.GradientTape() as tape:
logits = model(X)
 loss = compute_loss([Y, S], logits)
 gradients = tape.gradient(loss, model.trainable_variables)
 optimizer.apply_gradients(zip(gradients, model.trainable_variables))
train_loss(loss)
 def test_fat_step(inputs):
X, Y, S = inputs
logits = model(X)
 loss = compute_loss([Y, S], logits)
 test_loss(loss)
 def runs_fat(log_freq = 1):
for epoch in range(2):
for epoch_x, epoch_y, epoch_s in train_ds:
inputs = [epoch_x, epoch_y, epoch_s]
train_fat_step(inputs)
 for epoch_x, epoch_y, epoch_s in test_ds:
test_fat_step([epoch_x, epoch_y, epoch_s])
 template = 'epoch: {}, train_loss: {}, test_loss: {}'
print(template.format(epoch+1,
train_loss.result() ,
test_loss.result()))
runs_fat()
mother_sub_path = "/kaggle/input/submission"
train_sub_path = os.path.join(mother_sub_path, "submission.csv")
 submit = pd.read_csv(train_sub_path)
 print(submit)
 submit.to_csv('/kaggle/working/submission.csv',index=False )<import_modules>
data_dict['WNCAATourneySeeds']['Seed'] = data_dict['WNCAATourneySeeds']['Seed'].apply(lambda x: int(x[1:3]))
data_dict[fname].head()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import statsmodels.tsa.arima_model as arima
from sklearn.linear_model import LinearRegression
from sklearn import linear_model<set_options>
test = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WSampleSubmissionStage1_2020.csv')
 print(test.shape)
 test.head()
warnings.filterwarnings('ignore')
 pd.set_option("display.max_rows", None, "display.max_columns", None )<feature_engineering>
test = test.drop(['Pred'], axis=1)
 test['Season'] = test['ID'].apply(lambda x: int(x.split('_')[0]))
test['WTeamID'] = test['ID'].apply(lambda x: int(x.split('_')[1]))
test['LTeamID'] = test['ID'].apply(lambda x: int(x.split('_')[2]))
test.head()
training_data_original = pd.read_csv('/kaggle/input/inputs-data/train_data.csv')
 training_data_original['Date'] = pd.to_datetime(training_data_original['Date'])
 training_data = training_data_original[training_data_original['Country_Region'] != 'Diamond Princess']
countries = training_data['Country_Region'].unique()
countries_main = ['China', 'US', 'Australia', 'Canada']
states = training_data['Province_State'].unique()<load_from_csv>
gameCities = pd.merge(data_dict['WGameCities'], data_dict['Cities'], how='left', on=['CityID'])
 cols_to_use = gameCities.columns.difference(train.columns ).tolist() + ["Season", "WTeamID", "LTeamID"]
train = train.merge(gameCities[cols_to_use], how="left", on=["Season", "WTeamID", "LTeamID"])
 train.head()
cols_to_use = data_dict["WSeasons"].columns.difference(train.columns ).tolist() + ["Season"]
train = train.merge(data_dict["WSeasons"][cols_to_use], how="left", on=["Season"])
 train.head()
cols_to_use = data_dict["WTeams"].columns.difference(train.columns ).tolist()
train = train.merge(data_dict["WTeams"][cols_to_use], how="left", left_on=["WTeamID"], right_on=["TeamID"])
 train.drop(['TeamID'], axis=1, inplace=True)
 train = train.merge(data_dict["WTeams"][cols_to_use], how="left", left_on=["LTeamID"], right_on=["TeamID"], suffixes=('_W', '_L'))
train.drop(['TeamID'], axis=1, inplace=True)
 print(train.shape)
 train.head()
state_metadata = pd.read_excel('/kaggle/input/external-data-source/states_data.xlsx')
 country_metadata = pd.read_excel('/kaggle/input/external-data-source/countries_data.xlsx' )<groupby>
cols_to_use = data_dict['WNCAATourneySeeds'].columns.difference(train.columns ).tolist() + ['Season']
train = train.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=["Season","TeamID"]),
how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'])
 train.drop(['TeamID'], axis=1, inplace=True)
 train = train.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=["Season","TeamID"]),
how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], suffixes=('_W', '_L'))
train.drop(['TeamID'], axis=1, inplace=True)
 print(train.shape)
 train.head()
def data_preparation(training_data, countries, states, country_metadata, state_metadata, n_days_case,
n_days_fatal, min_num_cases = 2):
training_data_trun = training_data[training_data['ConfirmedCases'] >= min_num_cases]
conf_cases_dict = {}
fatal_dict = {}
for country in countries:
if country not in countries_main:
training_data_trun_loc = training_data_trun[(training_data_trun['Country_Region'] == country)&(pd.isnull(training_data_trun.Province_State)) ]
training_data_trun_loc = training_data_trun_loc.groupby(['Date'] ).sum()
training_data_trun_loc = training_data_trun_loc.sort_values(by = 'Date')
 if len(training_data_trun_loc['ConfirmedCases'].values)>= n_days_case:
conf_cases_dict[country] = training_data_trun_loc['ConfirmedCases'].values[:n_days_case] / country_metadata[country_metadata['Countries'] == country]['Population'].values[0]
if len(training_data_trun_loc['Fatalities'].values)>= n_days_fatal:
fatal_dict[country] = training_data_trun_loc['Fatalities'].values[:n_days_fatal]
for state in states:
training_data_trun_loc = training_data_trun[training_data_trun['Province_State'] == state]
training_data_trun_loc = training_data_trun_loc.groupby(['Date'] ).sum()
training_data_trun_loc = training_data_trun_loc.sort_values(by = 'Date')
 if len(training_data_trun_loc['ConfirmedCases'].values)>= n_days_case:
conf_cases_dict[state] = training_data_trun_loc['ConfirmedCases'].values[:n_days_case] / state_metadata[state_metadata['States'] == state]['Population'].values[0]
if len(training_data_trun_loc['Fatalities'].values)>= n_days_fatal:
fatal_dict[state] = training_data_trun_loc['Fatalities'].values[:n_days_fatal]
return pd.DataFrame(conf_cases_dict), pd.DataFrame(fatal_dict )<train_on_grid>
cols_to_use = gameCities.columns.difference(test.columns ).tolist() + ["Season", "WTeamID", "LTeamID"]
test = test.merge(gameCities[cols_to_use].drop_duplicates(subset=["Season", "WTeamID", "LTeamID"]),
how="left", on=["Season", "WTeamID", "LTeamID"])
 del gameCities
gc.collect()
test.head()
cols_to_use = data_dict["WSeasons"].columns.difference(test.columns ).tolist() + ["Season"]
test = test.merge(data_dict["WSeasons"][cols_to_use].drop_duplicates(subset=["Season"]),
how="left", on=["Season"])
 test.head()
cols_to_use = data_dict["WTeams"].columns.difference(test.columns ).tolist()
test = test.merge(data_dict["WTeams"][cols_to_use].drop_duplicates(subset=["TeamID"]),
how="left", left_on=["WTeamID"], right_on=["TeamID"])
 test.drop(['TeamID'], axis=1, inplace=True)
 test = test.merge(data_dict["WTeams"][cols_to_use].drop_duplicates(subset=["TeamID"]),
how="left", left_on=["LTeamID"], right_on=["TeamID"], suffixes=('_W', '_L'))
test.drop(['TeamID'], axis=1, inplace=True)
 test.head()
cols_to_use = data_dict['WNCAATourneySeeds'].columns.difference(test.columns ).tolist() + ['Season']
test = test.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=["Season","TeamID"]),
how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'TeamID'])
 test.drop(['TeamID'], axis=1, inplace=True)
 test = test.merge(data_dict['WNCAATourneySeeds'][cols_to_use].drop_duplicates(subset=["Season","TeamID"]),
how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'TeamID'], suffixes=('_W', '_L'))
test.drop(['TeamID'], axis=1, inplace=True)
 print(test.shape)
 test.head()
def fts_training(input_df, rank = 3):
matrix = input_df.values
u, s, v = np.linalg.svd(matrix, full_matrices=False)
 scores = np.matmul(u[:, :rank], np.diag(s[:rank]))
pcs = v[:rank, :]
return scores, pcs<create_dataframe>
not_exist_in_test = [c for c in train.columns.values.tolist() if c not in test.columns.values.tolist() ]
print(not_exist_in_test)
 train = train.drop(not_exist_in_test, axis=1)
 train.head()
def forecast_trajectories(training_data, countries, states, country_metadata,
state_metadata, loc = None, n_days_case = 30, n_days_fatal = 10,
forecast_days = 10, min_num_cases = 2, model_type = 'ARIMA',
components_modeling = True, rank = 3):
kf = KalmanFilter(initial_state_mean = np.zeros(rank ).tolist() , n_dim_obs = rank)
 conf_cases_df, fatal_df = data_preparation(training_data, countries, states,
country_metadata, state_metadata,
n_days_case = n_days_case,
n_days_fatal = n_days_case,
min_num_cases = min_num_cases)
 pred_countries = conf_cases_df.columns.tolist()
pred_countries_fatal = fatal_df.columns.tolist()
conf_cases_exog_df, fatal_exog_df = data_preparation(training_data, countries, states,
country_metadata, state_metadata,
n_days_case = n_days_case + forecast_days,
n_days_fatal = n_days_case + forecast_days,
min_num_cases = min_num_cases)
 scores_exog, pcs_exog = fts_training(conf_cases_exog_df, rank = rank)
 if len(scores_exog)> 0:
scores_exog = kf.em(scores_exog ).smooth(scores_exog)[0]
scores, pcs = fts_training(conf_cases_df, rank = rank)
 forecasted_scores = []
idx = 0
for score in scores.T:
if components_modeling:
exog = scores_exog[:n_days_case, idx] if len(scores_exog)> 0 else None
pred_exog = scores_exog[n_days_case:, idx] if len(scores_exog)> 0 else None
y = score
else:
exog = scores_exog[:n_days_case, :] if len(scores_exog)> 0 else None
pred_exog = scores_exog[n_days_case:, :] if len(scores_exog)> 0 else None
y = conf_cases_df[loc].values
try:
model = arima.ARIMA(endog = y, exog = exog, order =(4, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
try:
model = arima.ARIMA(endog = y, exog = exog, order =(3, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
try:
model = arima.ARIMA(endog = y, exog = exog, order =(2, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
model = arima.ARIMA(endog = y, exog = exog, order =(1, 0, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 if not components_modeling:
pred_traj = model.forecast(steps = forecast_days,
alpha = 0.001,
exog = pred_exog)[0]
break
else:
forecasted_scores.append(model.forecast(steps = forecast_days,
alpha = 0.001,
exog = pred_exog)[0].tolist())
 idx = idx + 1
if components_modeling:
pred_traj = np.matmul(np.array(forecasted_scores ).T, pcs)
 pred_traj_df = pd.DataFrame(pred_traj, columns = pred_countries)
 for loc in pred_countries:
if loc in country_metadata['Countries'].values.tolist() :
pred_traj_df[loc] = country_metadata[country_metadata['Countries']
== loc]['Population'].values[0] * pred_traj_df[loc]
if loc in state_metadata['States'].values.tolist() :
pred_traj_df[loc] = state_metadata[state_metadata['States']
== loc]['Population'].values[0] * pred_traj_df[loc]
else:
pred_traj_df = pd.DataFrame()
if loc in countries:
pred_traj_df[loc] = country_metadata[country_metadata['Countries']
== loc]['Population'].values[0] * pred_traj
elif loc in states:
pred_traj_df[loc] = state_metadata[state_metadata['States']
== loc]['Population'].values[0] * pred_traj
fatal_scores_exog, fatal_pcs_exog = fts_training(fatal_exog_df, rank = rank)
 if len(fatal_pcs_exog)> 0:
fatal_scores_exog = kf.em(fatal_scores_exog ).smooth(fatal_scores_exog)[0]
fatal_scores, fatal_pcs = fts_training(fatal_df, rank = rank)
 forecasted_fatal_scores = []
idx = 0
for fatal_score in fatal_scores.T:
if components_modeling:
exog = fatal_scores_exog[:n_days_fatal, idx] if len(fatal_scores_exog)> 0 else None
pred_exog = fatal_scores_exog[n_days_fatal:, idx] if len(fatal_scores_exog)> 0 else None
y = fatal_score
else:
exog = fatal_scores_exog[:n_days_fatal, :] if len(fatal_scores_exog)> 0 else None
pred_exog = fatal_scores_exog[n_days_fatal:, :] if len(fatal_scores_exog)> 0 else None
y = fatal_df[loc].values
try:
model = arima.ARIMA(endog = y, exog = exog, order =(4, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
try:
model = arima.ARIMA(endog = y, exog = exog, order =(3, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
try:
model = arima.ARIMA(endog = y, exog = exog, order =(2, 1, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 except:
model = arima.ARIMA(endog = y, exog = exog, order =(1, 0, 0)).fit(
 seasonal = False,
trace = False,
method = 'css',
solver = 'bfgs',
error_action = 'ignore',
setpwise_fit = True,
warn_convergence = True,
disp = False)
 if not components_modeling:
fatal_pred_traj = model.forecast(steps = forecast_days,
alpha = 0.001,
exog = pred_exog)[0]
break
else:
forecasted_fatal_scores.append(model.forecast(steps = forecast_days,
alpha = 0.001,
exog = pred_exog)[0].tolist())
 idx = idx + 1
if components_modeling:
fatal_pred_traj = np.matmul(np.array(forecasted_fatal_scores ).T, fatal_pcs)
 fatal_pred_traj_df = pd.DataFrame(fatal_pred_traj, columns = pred_countries_fatal)
 else:
fatal_pred_traj_df = pd.DataFrame()
if loc in countries:
fatal_pred_traj_df[loc] = country_metadata[country_metadata['Countries']
== loc]['Population'].values[0] * fatal_pred_traj
elif loc in states:
fatal_pred_traj_df[loc] = state_metadata[state_metadata['States']
== loc]['Population'].values[0] * fatal_pred_traj
return pred_traj_df, fatal_pred_traj_df<predict_on_test>
team_win_score = regularSeason.groupby(['Season', 'WTeamID'] ).agg({'WScore':['sum', 'count', 'var']} ).reset_index()
team_win_score.columns = [' '.join(col ).strip() for col in team_win_score.columns.values]
team_loss_score = regularSeason.groupby(['Season', 'LTeamID'] ).agg({'LScore':['sum', 'count', 'var']} ).reset_index()
team_loss_score.columns = [' '.join(col ).strip() for col in team_loss_score.columns.values]
del regularSeason
gc.collect()
pred_traj_df, fatal_pred_traj_df = forecast_trajectories(training_data, countries, states, country_metadata, state_metadata,
loc = 'New York', n_days_case = 55, n_days_fatal = 55,
forecast_days = 33, rank = 3, min_num_cases = 2,
components_modeling = True )<filter>
train = pd.merge(train, team_win_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'WTeamID'])
 train = pd.merge(train, team_loss_score, how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'LTeamID'])
 train = pd.merge(train, team_loss_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'LTeamID'])
 train = pd.merge(train, team_win_score, how='left', left_on=['Season', 'LTeamID_x'], right_on=['Season', 'WTeamID'])
 train.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)
 train.head()
def generate_prediction(test_data, training_data, countries_main, countries, states, min_num_cases):
for country in countries:
print(country)
 if country not in countries_main and country not in excl_list:
test_loc_df = test_data[(test_data['Country_Region'] == country)&(pd.isnull(test_data.Province_State)) ].reset_index()
train_loc_df = training_data[(training_data['Country_Region'] == country)&(pd.isnull(training_data.Province_State)) ].reset_index()
test_start = test_loc_df['Date'][0]
test_end = test_loc_df['Date'][len(test_loc_df)- 1]
train_end = train_loc_df['Date'][len(train_loc_df)- 1]
test_loc_df.loc[(( test_loc_df['Date'] >= test_start)) &(test_loc_df['Date'] <= train_end), 'ConfirmedCases'] = train_loc_df[(train_loc_df['Date'] >= test_start)&(train_loc_df['Date'] <= train_end)]['ConfirmedCases'].values
test_loc_df.loc[(( test_loc_df['Date'] >= test_start)) &(test_loc_df['Date'] <= train_end), 'Fatalities'] = train_loc_df[(train_loc_df['Date'] >= test_start)&(train_loc_df['Date'] <= train_end)]['Fatalities'].values
effective_df = train_loc_df[train_loc_df['ConfirmedCases'] >= min_num_cases]
forecast_days = int(( test_end - train_end ).days)
 if len(effective_df)> 0:
effective_train_start = train_loc_df[train_loc_df['ConfirmedCases'] >= min_num_cases].reset_index() ['Date'][0]
n_days_case = int(( train_end - effective_train_start ).days)+ 1
pred_df, fatal_pred_df = forecast_trajectories(training_data, countries, states,
country_metadata, state_metadata, loc = country,
n_days_case = n_days_case, n_days_fatal = n_days_case,
forecast_days = forecast_days, min_num_cases = min_num_cases,
rank = 3)
 test_loc_df.loc[test_loc_df['Date'] > train_end, 'ConfirmedCases'] = np.maximum.accumulate(pred_df[country].values ).astype(int)
 test_data.loc[(test_data['Country_Region'] == country)&(pd.isnull(test_data.Province_State)) , 'ConfirmedCases'] = test_loc_df['ConfirmedCases'].values
test_loc_df.loc[test_loc_df['Date'] > train_end, 'Fatalities'] = np.maximum.accumulate(fatal_pred_df[country].values ).astype(int)
 test_data.loc[(test_data['Country_Region'] == country)&(pd.isnull(test_data.Province_State)) , 'Fatalities'] = test_loc_df['Fatalities'].values
for state in states:
print(state)
 if str(state)not in excl_list:
test_loc_df = test_data[(test_data['Province_State'] == state)].reset_index()
train_loc_df = training_data[(training_data['Province_State'] == state)].reset_index()
test_start = test_loc_df['Date'][0]
test_end = test_loc_df['Date'][len(test_loc_df)- 1]
train_end = train_loc_df['Date'][len(train_loc_df)- 1]
test_loc_df.loc[(( test_loc_df['Date'] >= test_start)) &(test_loc_df['Date'] <= train_end), 'ConfirmedCases'] = train_loc_df[(train_loc_df['Date'] >= test_start)&(train_loc_df['Date'] <= train_end)]['ConfirmedCases'].values
test_loc_df.loc[(( test_loc_df['Date'] >= test_start)) &(test_loc_df['Date'] <= train_end), 'Fatalities'] = train_loc_df[(train_loc_df['Date'] >= test_start)&(train_loc_df['Date'] <= train_end)]['Fatalities'].values
effective_df = train_loc_df[train_loc_df['ConfirmedCases'] >= min_num_cases]
forecast_days = int(( test_end - train_end ).days)
 if len(effective_df)> 0:
effective_train_start = train_loc_df[train_loc_df['ConfirmedCases'] >= min_num_cases].reset_index() ['Date'][0]
n_days_case = int(( train_end - effective_train_start ).days)+ 1
pred_df, fatal_pred_df = forecast_trajectories(training_data, countries, states,
country_metadata, state_metadata, loc = state,
n_days_case = n_days_case, n_days_fatal = n_days_case,
forecast_days = forecast_days, min_num_cases = min_num_cases,
rank = 3)
 test_loc_df.loc[test_loc_df['Date'] > train_end, 'ConfirmedCases'] = np.maximum.accumulate(pred_df[state].values ).astype(int)
 test_data.loc[test_data['Province_State'] == state, 'ConfirmedCases'] = test_loc_df['ConfirmedCases'].values
test_loc_df.loc[test_loc_df['Date'] > train_end, 'Fatalities'] = np.maximum.accumulate(fatal_pred_df[state].values ).astype(int)
 test_data.loc[test_data['Province_State'] == state, 'Fatalities'] = test_loc_df['Fatalities'].values
return test_data<load_from_csv>
test = pd.merge(test, team_win_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'WTeamID'])
 test = pd.merge(test, team_loss_score, how='left', left_on=['Season', 'LTeamID'], right_on=['Season', 'LTeamID'])
 test = pd.merge(test, team_loss_score, how='left', left_on=['Season', 'WTeamID'], right_on=['Season', 'LTeamID'])
 test = pd.merge(test, team_win_score, how='left', left_on=['Season', 'LTeamID_x'], right_on=['Season', 'WTeamID'])
 test.drop(['LTeamID_y', 'WTeamID_y'], axis=1, inplace=True)
 test.head()
test_data = pd.read_csv('/kaggle/input/inputs-data/test_data.csv')
 test_data['Date'] = pd.to_datetime(test_data['Date'] )<feature_engineering>
def preprocess(df):
df['x_score'] = df['WScore sum_x'] + df['LScore sum_y']
df['y_score'] = df['WScore sum_y'] + df['LScore sum_x']
df['x_count'] = df['WScore count_x'] + df['LScore count_y']
df['y_count'] = df['WScore count_y'] + df['WScore count_x']
df['x_var'] = df['WScore var_x'] + df['LScore count_y']
df['y_var'] = df['WScore var_y'] + df['WScore var_x']
return df
train = preprocess(train)
 test = preprocess(test )
test_data['ConfirmedCases'] = None
test_data['Fatalities'] = None<define_variables>
train_win = train.copy()
train_los = train.copy()
train_win = train_win[['Seed_W', 'Seed_L', 'TeamName_W', 'TeamName_L',
'x_score', 'y_score', 'x_count', 'y_count', 'x_var', 'y_var']]
train_los = train_los[['Seed_L', 'Seed_W', 'TeamName_L', 'TeamName_W',
'y_score', 'x_score', 'x_count', 'y_count', 'x_var', 'y_var']]
train_win.columns = ['Seed_1', 'Seed_2', 'TeamName_1', 'TeamName_2',
'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']
train_los.columns = ['Seed_1', 'Seed_2', 'TeamName_1', 'TeamName_2',
'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']
test = test[['ID', 'Seed_W', 'Seed_L', 'TeamName_W', 'TeamName_L',
'x_score', 'y_score', 'x_count', 'y_count', 'x_var', 'y_var']]
test.columns = ['ID', 'Seed_1', 'Seed_2', 'TeamName_1', 'TeamName_2',
'Score_1', 'Score_2', 'Count_1', 'Count_2', 'Var_1', 'Var_2']
excl_list = ['Hubei', 'Belize', 'Chad', 'Eritrea', 'Grenada', 'Guinea-Bissau', 'Holy See',
'Laos', 'Libya', 'Mali', 'Mozambique', 'Saint Kitts and Nevis', 'Somalia',
'Syria', 'nan', 'Saint Barthelemy', 'Virgin Islands', 'Montserrat',
'Congo(Brazzaville)', 'Gabon', 'Guinea', 'Guyana', 'Haiti', 'Namibia',
'Saint Lucia', 'Seychelles', 'Curacao', 'Cayman Islands']<predict_on_test>
def feature_engineering(df):
df['Seed_diff'] = df['Seed_1'] - df['Seed_2']
df['Score_diff'] = df['Score_1'] - df['Score_2']
df['Count_diff'] = df['Count_1'] - df['Count_2']
df['Var_diff'] = df['Var_1'] - df['Var_2']
df['Mean_score1'] = df['Score_1'] / df['Count_1']
df['Mean_score2'] = df['Score_2'] / df['Count_2']
df['Mean_score_diff'] = df['Mean_score1'] - df['Mean_score2']
df['FanoFactor_1'] = df['Var_1'] / df['Mean_score1']
df['FanoFactor_2'] = df['Var_2'] / df['Mean_score2']
return df
train_win = feature_engineering(train_win)
 train_los = feature_engineering(train_los)
 test = feature_engineering(test )
predictions = generate_prediction(test_data, training_data, countries_main, countries, states, min_num_cases = 10 )<count_unique_values>
data = pd.concat(( train_win, train_los)).reset_index(drop=True)
 print(data.shape)
 data.head()
missing_countries = predictions[(predictions.ConfirmedCases.isnull())&(pd.isnull(predictions.Province_State)) ]['Country_Region'].unique().tolist()
missing_states = predictions[(predictions.ConfirmedCases.isnull())&(pd.notnull(predictions.Province_State)) ]['Province_State'].unique().tolist()<split>
categoricals = ["TeamName_1", "TeamName_2"]
for c in categoricals:
le = LabelEncoder()
data[c] = data[c].fillna("NaN")
 data[c] = le.fit_transform(data[c])
 test[c] = le.transform(test[c])
 data.head()
def fill_excl_pred(predictions, training_data_original, test_data):
for country in missing_countries:
print(country)
 pred_loc_df = predictions[predictions['Country_Region'] == country]
train_loc_df = training_data_original[training_data_original['Country_Region'] == country]
series_comf_cases = train_loc_df['ConfirmedCases'].values
series_fatal = train_loc_df['Fatalities'].values
test_start = test_data[test_data['Country_Region'] == country]['Date'].values[0]
series_comf_cases_test = training_data_original[(training_data_original['Country_Region'] == country)&(training_data_original['Date'] >= test_start)]['ConfirmedCases']
series_fatal_test = training_data_original[(training_data_original['Country_Region'] == country)&(training_data_original['Date'] >= test_start)]['Fatalities']
if len(series_comf_cases)> 0:
regressor = LinearRegression()
regressor.fit(np.arange(len(series_comf_cases_test)).reshape(-1, 1), series_comf_cases_test)
 comf_cases_pred = regressor.predict(np.arange(10, 40 ).reshape(-1, 1))
regressor.fit(np.arange(len(series_fatal_test)).reshape(-1, 1), series_fatal_test)
 fatal_pred = regressor.predict(np.arange(10, 40 ).reshape(-1, 1))
else:
comf_cases_pred = []
fatal_pred = []
conf_cases_loc = np.concatenate(( series_comf_cases_test, comf_cases_pred), axis=0)
 fatal_loc = np.concatenate(( series_fatal_test, fatal_pred), axis=0)
 predictions.loc[predictions['Country_Region'] == country, 'ConfirmedCases'] = conf_cases_loc.astype(int)
 predictions.loc[predictions['Country_Region'] == country, 'Fatalities'] = fatal_loc.astype(int)
 for state in missing_states:
print(state)
 pred_loc_df = predictions[predictions['Province_State'] == state]
train_loc_df = training_data_original[training_data_original['Province_State'] == state]
series_comf_cases = train_loc_df['ConfirmedCases'].values
series_fatal = train_loc_df['Fatalities'].values
test_start = test_data[test_data['Province_State'] == state]['Date'].values[0]
series_comf_cases_test = training_data_original[(training_data_original['Province_State'] == state)&(training_data_original['Date'] >= test_start)]['ConfirmedCases']
series_fatal_test = training_data[(training_data_original['Province_State'] == state)&(training_data_original['Date'] >= test_start)]['Fatalities']
regressor = LinearRegression()
regressor.fit(np.arange(len(series_comf_cases_test)).reshape(-1, 1), series_comf_cases_test)
 comf_cases_pred = regressor.predict(np.arange(10, 40 ).reshape(-1, 1))
regressor.fit(np.arange(len(series_fatal_test)).reshape(-1, 1), series_fatal_test)
 fatal_pred = regressor.predict(np.arange(10, 40 ).reshape(-1, 1))
conf_cases_loc = np.concatenate(( series_comf_cases_test, comf_cases_pred), axis=0)
 fatal_loc = np.concatenate(( series_fatal_test, fatal_pred), axis=0)
 predictions.loc[predictions['Province_State'] == state, 'ConfirmedCases'] = conf_cases_loc.astype(int)
 predictions.loc[predictions['Province_State'] == state, 'Fatalities'] = fatal_loc.astype(int)
 return predictions<predict_on_test>
target = 'result'
features = data.columns.values.tolist()
features.remove(target )
prediction_final = fill_excl_pred(predictions, training_data_original, test_data )<create_dataframe>
lgbm = LgbModel(data, test, target, features, categoricals=categoricals, n_splits=10,
cv_method="StratifiedKFold", group=None, task="classification", scaler=None, verbose=True )
predictions_csv = pd.DataFrame()<save_to_csv>
submission_df = pd.read_csv('.. /input/google-cloud-ncaa-march-madness-2020-division-1-womens-tournament/WSampleSubmissionStage1_2020.csv')
 submission_df['Pred'] = lgbm.y_pred
submission_df
<import_modules><EOS> 
submission_df.to_csv('submission.csv', index=False )