Kaggle competition: COVID19 Global Forecasting (Week 2)
train_X = pd.read_csv('.. /input/all-features/train_X.csv')
 test_X = pd.read_csv('.. /input/all-features/test_X.csv')
 train_y = pd.read_csv('.. /input/all-features/train_y.csv' )<load_from_csv>
pd.set_option('display.max_columns', 100)
 warnings.filterwarnings('ignore' )
submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv', index_col='seg_id')
 submission.shape<choose_model_class>
df_train = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 print(df_train.shape)
 print(df_train.Date.min() , df_train.Date.max())
 df_train.head()
mse_scoring = make_scorer(mean_squared_error, greater_is_better=False)
 mae_scoring = make_scorer(mean_absolute_error, greater_is_better=False)
<train_model>
train_min_date, train_max_date = df_train.Date.min() , df_train.Date.max()
train_min_dayofyear, train_max_dayofyear =(pd.to_datetime(train_min_date)).dayofyear,(pd.to_datetime(train_max_date)).dayofyear
print(train_min_dayofyear, train_max_dayofyear )
def run_cat(
 X_tr: PandasDataFrame,
X_val: PandasDataFrame,
y_tr: PandasDataFrame,
y_val: PandasDataFrame,
test_data: PandasDataFrame,
params: Dict 
):

 early_stop = 200
num_rounds = 10000
verbose_eval=1000
d_train = cat.Pool(X_tr, label=y_tr)
 d_valid = cat.Pool(X_val, label=y_val)
 d_predict = cat.Pool(test_data)
 cat_model = cat.CatBoostRegressor(**params)
 cat_model.fit(
 X=d_train,use_best_model=True,
eval_set=d_valid,
early_stopping_rounds=early_stop 
)
 val_pred = cat_model.predict(d_valid)
 prediction = cat_model.predict(d_predict)
 return val_pred, prediction, cat_model.get_feature_importance()
def run_cv_model(
 train_data,
train_target,
test_data,
model_fn, params,
scoring,
label='model',
feature_imp=False,
n_folds = 8 
):
oof_val = np.zeros(len(train_data))
predictions = np.zeros(len(test_data))
oof_predict = np.zeros(( n_folds, test_data.shape[0]))
scores = []
feature_importance_df = pd.DataFrame()
folds = KFold(n_splits=n_folds, shuffle=True, random_state=42)
 train_columns = train_data.columns.values
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_data, train_target.values)) :
strLog = "fold {} for {}".format(fold_, label)
 print(strLog)
 X_tr, X_val = train_data.iloc[trn_idx], train_data.iloc[val_idx]
y_tr, y_val = train_target.iloc[trn_idx], train_target.iloc[val_idx]
val_pred, prediction, feature_importances = model_fn(
 X_tr, X_val,
y_tr, y_val,
test_data,
params 
)
 score = scoring(y_val, val_pred)
 scores.append(score)
 oof_val[val_idx] = val_pred
if feature_imp == True:
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = train_columns
fold_importance_df["importance"] = feature_importances[:len(train_columns)]
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 oof_predict[fold_] = prediction
predictions += prediction/folds.n_splits
print('CV score: {0:.4f}, std: {1:.4f}.
 '.format(np.mean(score), np.std(score)))
 print('CV mean score: {0:.4f}, std: {1:.4f}.
 '.format(np.mean(scores), np.std(scores)))
 return oof_val, oof_predict, predictions, feature_importance_df, scores<choose_model_class>
train_valid_cutoff_dayofyear = train_min_dayofyear +(train_max_dayofyear - train_min_dayofyear)// 3 * 2
train_valid_cutoff_dayofyear
def cat_bayesian(**cat_bayes_params):
cat_bayes_params['iterations'] = int(cat_bayes_params['iterations'])
 cat_bayes_params['depth'] = int(cat_bayes_params['depth'])
 cat_bayes_params['border_count'] = int(cat_bayes_params['border_count'])
 cat_bayes_params['l2_leaf_reg'] = int(cat_bayes_params['l2_leaf_reg'])
 cat_bayes_params['one_hot_max_size'] = int(cat_bayes_params['one_hot_max_size'])
 cat_bayes_params['max_ctr_complexity'] = int(cat_bayes_params['max_ctr_complexity'])
 cat_bayes_params['random_seed'] = int(cat_bayes_params['random_seed'])
 fix_cat_params = {
'task_type':"GPU",
'loss_function':'MAE',
'eval_metric':'MAE',
'verbose': False
}
cat_params = {**fix_cat_params, **cat_bayes_params}
oof_val, oof_predict, predictions, feature_importance_df, scores = run_cv_model(
 train_data=train_X,
train_target=train_y,
test_data=test_X,
model_fn=run_cat,
params=cat_params,
scoring=mean_absolute_error,
label="cat",
feature_imp=True,
n_folds=5 
)
 return -np.mean(scores )<choose_model_class>
df_test = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 print(df_test.shape)
 test_min_date, test_max_date = df_test.Date.min() , df_test.Date.max()
print(test_min_date, test_max_date)
 df_test.head()
search_spaces = {
'iterations':(200, 500),
'depth':(1, 9),
'learning_rate':(0.01, 1.0),
'random_strength':(1e-9, 10),
'bagging_temperature':(0.0, 1.0),
'border_count':(1, 100),
'l2_leaf_reg':(2, 50),
'one_hot_max_size':(0, 20),
'max_ctr_complexity':(1, 3),
'random_seed':(1,81)
 }
cat_bo = BayesianOptimization(cat_bayesian,search_spaces, random_state=9)
<set_options>
df_traintest = pd.concat([df_train, df_test])
 print(df_train.shape, df_test.shape, df_traintest.shape)
 df_traintest.head()
with warnings.catch_warnings() :
warnings.filterwarnings('ignore')
 cat_bo.maximize(init_points=10, n_iter=10, acq='ucb' )<init_hyperparams>
def func(x):
try:
x_new = x['Country_Region'] + "/" + x['Province_State']
except:
x_new = x['Country_Region']
return x_new
df_traintest['place_id'] = df_traintest.apply(lambda x: func(x), axis=1)
 tmp = np.sort(df_traintest['place_id'].unique())
 print("num unique places: {}".format(len(tmp)))
 print(tmp[:10] )
cat_bayes_params = cat_bo.max['params']
cat_bayes_params['iterations'] = int(cat_bayes_params['iterations'])
 cat_bayes_params['depth'] = int(cat_bayes_params['depth'])
 cat_bayes_params['border_count'] = int(cat_bayes_params['border_count'])
 cat_bayes_params['l2_leaf_reg'] = int(cat_bayes_params['l2_leaf_reg'])
 cat_bayes_params['one_hot_max_size'] = int(cat_bayes_params['one_hot_max_size'])
 cat_bayes_params['max_ctr_complexity'] = int(cat_bayes_params['max_ctr_complexity'])
 cat_bayes_params['random_seed'] = int(cat_bayes_params['random_seed'] )<import_modules>
df_traintest['Date'] = pd.to_datetime(df_traintest['Date'])
 time_cols = [
"month", "week", "day",
"dayofyear", "dayofweek",
]
for attr in time_cols:
dtype = np.int16 if attr == "year" else np.int8
df_traintest[attr] = getattr(df_traintest['Date'].dt, attr ).astype(dtype)
 df_traintest["is_weekend"] = df_traintest["dayofweek"].isin([5, 6] ).astype(np.int8)
 time_cols += ["is_weekend"]
print(time_cols)
 df_traintest.head(10 )
class FeatureSelector() :

 def __init__(self, data, labels=None):
self.data = data
self.labels = labels
if labels is None:
print('No labels provided.Feature importance based methods are not available.')
 self.base_features = list(data.columns)
 self.one_hot_features = None
self.record_missing = None
self.record_single_unique = None
self.record_collinear = None
self.record_zero_importance = None
self.record_low_importance = None
self.missing_stats = None
self.unique_stats = None
self.corr_matrix = None
self.feature_importances = None
self.ops = {}
self.one_hot_correlated = False
def identify_missing(self, missing_threshold):

 self.missing_threshold = missing_threshold
missing_series = self.data.isnull().sum() / self.data.shape[0]
self.missing_stats = pd.DataFrame(missing_series ).rename(columns = {'index': 'feature', 0: 'missing_fraction'})
 self.missing_stats = self.missing_stats.sort_values('missing_fraction', ascending = False)
 record_missing = pd.DataFrame(missing_series[missing_series > missing_threshold] ).reset_index().rename(columns =
{'index': 'feature',
0: 'missing_fraction'})
 to_drop = list(record_missing['feature'])
 self.record_missing = record_missing
self.ops['missing'] = to_drop
print('%d features with greater than %0.2f missing values.
 ' %(len(self.ops['missing']), self.missing_threshold))
def identify_single_unique(self):

 unique_counts = self.data.nunique()
self.unique_stats = pd.DataFrame(unique_counts ).rename(columns = {'index': 'feature', 0: 'nunique'})
 self.unique_stats = self.unique_stats.sort_values('nunique', ascending = True)
 record_single_unique = pd.DataFrame(unique_counts[unique_counts == 1] ).reset_index().rename(columns = {'index': 'feature',
0: 'nunique'})
 to_drop = list(record_single_unique['feature'])
 self.record_single_unique = record_single_unique
self.ops['single_unique'] = to_drop
print('%d features with a single unique value.
 ' % len(self.ops['single_unique']))
def identify_collinear(self, correlation_threshold, one_hot=False):

 self.correlation_threshold = correlation_threshold
self.one_hot_correlated = one_hot
if one_hot:
features = pd.get_dummies(self.data)
 self.one_hot_features = [column for column in features.columns if column not in self.base_features]
self.data_all = pd.concat([features[self.one_hot_features], self.data], axis = 1)
 corr_matrix = pd.get_dummies(features ).corr()
else:
corr_matrix = self.data.corr()
self.corr_matrix = corr_matrix
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k = 1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(upper[column].abs() > correlation_threshold)]
record_collinear = pd.DataFrame(columns = ['drop_feature', 'corr_feature', 'corr_value'])
 for column in to_drop:
corr_features = list(upper.index[upper[column].abs() > correlation_threshold])
 corr_values = list(upper[column][upper[column].abs() > correlation_threshold])
 drop_features = [column for _ in range(len(corr_features)) ]
temp_df = pd.DataFrame.from_dict({'drop_feature': drop_features,
'corr_feature': corr_features,
'corr_value': corr_values})
 record_collinear = record_collinear.append(temp_df, ignore_index = True)
 self.record_collinear = record_collinear
self.ops['collinear'] = to_drop
print('%d features with a correlation magnitude greater than %0.2f.
 ' %(len(self.ops['collinear']), self.correlation_threshold))
def identify_zero_importance(self, feature_importances):

 if self.labels is None:
raise ValueError("No training labels provided.")
 features = pd.get_dummies(self.data)
 self.one_hot_features = [column for column in features.columns if column not in self.base_features]
self.data_all = pd.concat([features[self.one_hot_features], self.data], axis = 1)
 feature_names = list(features.columns)
 features = np.array(features)
 labels = np.array(self.labels ).reshape(( -1,))
feature_importances = feature_importances.sort_values('importance', ascending = False ).reset_index(drop = True)
 feature_importances['normalized_importance'] = feature_importances['importance'] / feature_importances['importance'].sum()
feature_importances['cumulative_importance'] = np.cumsum(feature_importances['normalized_importance'])
 record_zero_importance = feature_importances[feature_importances['importance'] == 0.0]
to_drop = list(record_zero_importance['feature'])
 self.feature_importances = feature_importances
self.record_zero_importance = record_zero_importance
self.ops['zero_importance'] = to_drop
print('
%d features with zero importance after one-hot encoding.
 ' % len(self.ops['zero_importance']))
def identify_low_importance(self, cumulative_importance):

 self.cumulative_importance = cumulative_importance
if self.feature_importances is None:
raise NotImplementedError()
self.feature_importances = self.feature_importances.sort_values('cumulative_importance')
 record_low_importance = self.feature_importances[self.feature_importances['cumulative_importance'] > cumulative_importance]
to_drop = list(record_low_importance['feature'])
 self.record_low_importance = record_low_importance
self.ops['low_importance'] = to_drop
print('%d features required for cumulative importance of %0.2f after one hot encoding.' %(len(self.feature_importances)-
len(self.record_low_importance), self.cumulative_importance))
print('%d features do not contribute to cumulative importance of %0.2f.
 ' %(len(self.ops['low_importance']),
self.cumulative_importance))
def identify_all(self, selection_params):

 for param in ['missing_threshold', 'correlation_threshold', 'eval_metric', 'task', 'cumulative_importance']:
if param not in selection_params.keys() :
raise ValueError('%s is a required parameter for this method.' % param)
 self.identify_missing(selection_params['missing_threshold'])
 self.identify_single_unique()
self.identify_collinear(selection_params['correlation_threshold'])
 self.identify_zero_importance(task = selection_params['task'], eval_metric = selection_params['eval_metric'])
 self.identify_low_importance(selection_params['cumulative_importance'])
 self.all_identified = set(list(chain(*list(self.ops.values()))))
self.n_identified = len(self.all_identified)
 print('%d total features out of %d identified for removal after one-hot encoding.
 ' %(self.n_identified,
self.data_all.shape[1]))
def check_removal(self, keep_one_hot=True):

 self.all_identified = set(list(chain(*list(self.ops.values()))))
print('Total of %d features identified for removal' % len(self.all_identified))
if not keep_one_hot:
if self.one_hot_features is None:
print('Data has not been one-hot encoded')
 else:
one_hot_to_remove = [x for x in self.one_hot_features if x not in self.all_identified]
print('%d additional one-hot features can be removed' % len(one_hot_to_remove))
return list(self.all_identified)
 def remove(self, methods, keep_one_hot = True):

 features_to_drop = []
if methods == 'all':
data = self.data_all
print('{} methods have been run
'.format(list(self.ops.keys())))
 features_to_drop = set(list(chain(*list(self.ops.values()))))
else:
if 'zero_importance' in methods or 'low_importance' in methods or self.one_hot_correlated:
data = self.data_all
else:
data = self.data
for method in methods:
if method not in self.ops.keys() :
raise NotImplementedError('%s method has not been run' % method)
 else:
features_to_drop.append(self.ops[method])
 features_to_drop = set(list(chain(*features_to_drop)))
 features_to_drop = list(features_to_drop)
 if not keep_one_hot:
if self.one_hot_features is None:
print('Data has not been one-hot encoded')
 else:
features_to_drop = list(set(features_to_drop)| set(self.one_hot_features))
data = data.drop(columns = features_to_drop)
 self.removed_features = features_to_drop
if not keep_one_hot:
	print('Removed %d features including one-hot features.' % len(features_to_drop))
else:
	print('Removed %d features.' % len(features_to_drop))
return data
def plot_missing(self):

 if self.record_missing is None:
raise NotImplementedError("Missing values have not been calculated.Run `identify_missing`")
 self.reset_plot()
plt.style.use('seaborn-white')
 plt.figure(figsize =(7, 5))
plt.hist(self.missing_stats['missing_fraction'], bins = np.linspace(0, 1, 11), edgecolor = 'k', color = 'red', linewidth = 1.5)
 plt.xticks(np.linspace(0, 1, 11)) ;
plt.xlabel('Missing Fraction', size = 14); plt.ylabel('Count of Features', size = 14);
plt.title("Fraction of Missing Values Histogram", size = 16);
def plot_unique(self):

 if self.record_single_unique is None:
raise NotImplementedError('Unique values have not been calculated.Run `identify_single_unique`')
 self.reset_plot()
self.unique_stats.plot.hist(edgecolor = 'k', figsize =(7, 5))
plt.ylabel('Frequency', size = 14); plt.xlabel('Unique Values', size = 14);
plt.title('Number of Unique Values Histogram', size = 16);
def plot_collinear(self, plot_all = False):

 if self.record_collinear is None:
raise NotImplementedError('Collinear features have not been idenfitied.Run `identify_collinear`.')
 if plot_all:
	corr_matrix_plot = self.corr_matrix
	title = 'All Correlations'
else:
	
	
	 corr_matrix_plot = self.corr_matrix.loc[list(set(self.record_collinear['corr_feature'])) ,
	 list(set(self.record_collinear['drop_feature'])) ]
	 title = "Correlations Above Threshold"
f, ax = plt.subplots(figsize=(10, 8))
cmap = sns.diverging_palette(220, 10, as_cmap=True)
 sns.heatmap(corr_matrix_plot, cmap=cmap, center=0,
linewidths=.25, cbar_kws={"shrink": 0.6})
 ax.set_yticks([x + 0.5 for x in list(range(corr_matrix_plot.shape[0])) ])
 ax.set_yticklabels(list(corr_matrix_plot.index), size = int(160 / corr_matrix_plot.shape[0])) ;
ax.set_xticks([x + 0.5 for x in list(range(corr_matrix_plot.shape[1])) ])
 ax.set_xticklabels(list(corr_matrix_plot.columns), size = int(160 / corr_matrix_plot.shape[1])) ;
plt.title(title, size = 14)
 def plot_feature_importances(self, plot_n = 15, threshold = None):

 if self.record_zero_importance is None:
raise NotImplementedError('Feature importances have not been determined.Run `idenfity_zero_importance`')
 if plot_n > self.feature_importances.shape[0]:
plot_n = self.feature_importances.shape[0] - 1
self.reset_plot()
plt.figure(figsize =(10, 6))
ax = plt.subplot()
ax.barh(list(reversed(list(self.feature_importances.index[:plot_n]))),
self.feature_importances['normalized_importance'][:plot_n],
align = 'center', edgecolor = 'k')
 ax.set_yticks(list(reversed(list(self.feature_importances.index[:plot_n]))))
ax.set_yticklabels(self.feature_importances['feature'][:plot_n], size = 12)
 plt.xlabel('Normalized Importance', size = 16); plt.title('Feature Importances', size = 18)
 plt.show()
plt.figure(figsize =(6, 4))
plt.plot(list(range(1, len(self.feature_importances)+ 1)) , self.feature_importances['cumulative_importance'], 'r-')
 plt.xlabel('Number of Features', size = 14); plt.ylabel('Cumulative Importance', size = 14);
plt.title('Cumulative Feature Importance', size = 16);
if threshold:
importance_index = np.min(np.where(self.feature_importances['cumulative_importance'] > threshold))
plt.vlines(x = importance_index + 1, ymin = 0, ymax = 1, linestyles='--', colors = 'blue')
 plt.show() ;
print('%d features required for %0.2f of cumulative importance' %(importance_index + 1, threshold))
def reset_plot(self):
plt.rcParams = plt.rcParamsDefault<train_on_grid>
df_traintest['cases/day'] = 0
df_traintest['fatal/day'] = 0
places = np.sort(df_traintest['place_id'].unique())
 for place in places:
tmp = df_traintest['ConfirmedCases'][df_traintest['place_id']==place].values
tmp[1:] -= tmp[:-1]
df_traintest['cases/day'][df_traintest['place_id']==place] = tmp
tmp = df_traintest['Fatalities'][df_traintest['place_id']==place].values
tmp[1:] -= tmp[:-1]
df_traintest['fatal/day'][df_traintest['place_id']==place] = tmp
df_traintest[df_traintest['place_id']=='China/Hubei']
fix_cat_params = {
'task_type':"GPU",
'loss_function':'MAE',
'eval_metric':'MAE',
'verbose': False
}
cat_params = {**fix_cat_params, **cat_bayes_params}
oof_val, oof_predict, predictions, feature_importance_df, scores = run_cv_model(
 train_data=train_X,
train_target=train_y,
test_data=test_X,
model_fn=run_cat,
params=cat_params,
scoring=mean_absolute_error,
label="cat",
feature_imp=True,
n_folds=8
)<save_to_csv>
def do_aggregation(df, col, mean_range, method='mean'):
df_new = copy.deepcopy(df)
 col_new = '{}_{}_({}-{})'.format(col, method, mean_range[0], mean_range[1])
 df_new[col_new] = 0
if method=='mean':
tmp = df_new[col].rolling(mean_range[1]-mean_range[0]+1 ).mean()
elif method=='std':
tmp = df_new[col].rolling(mean_range[1]-mean_range[0]+1 ).std()
df_new[col_new][mean_range[0]:] = tmp[:-(mean_range[0])]
df_new[col_new][pd.isna(df_new[col_new])] = 0
return df_new[[col_new]].reset_index(drop=True)
 def do_aggregations(df):
for method in ['mean']:
df = pd.concat([df, do_aggregation(df, 'cases/day', [1,1], method ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'cases/day', [1,7], method ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'cases/day', [8,14], method ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [1,1], method ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [1,7], method ).reset_index(drop=True)], axis=1)
 df = pd.concat([df, do_aggregation(df, 'fatal/day', [8,14], method ).reset_index(drop=True)], axis=1)
 return df
submission.to_csv('submission.csv',index=True)
 fs = FeatureSelector(data = train_X, labels = train_y)
 feature_importance = feature_importance_df[["feature", "importance"]].groupby("feature" ).mean()
feature_importance['feature'] = feature_importance.index<define_variables>
df_traintest2 = []
for place in places[:]:
df_tmp = df_traintest[df_traintest['place_id']==place].reset_index(drop=True)
 df_tmp = do_aggregations(df_tmp)
 df_traintest2.append(df_tmp)
 df_traintest2 = pd.concat(df_traintest2 ).reset_index(drop=True)
 df_traintest2[df_traintest2['place_id']=='China/Hubei'].head(20 )
one_hot_features = fs.one_hot_features
base_features = fs.base_features
print('There are %d original features' % len(base_features))
print('There are %d one-hot features' % len(one_hot_features))<drop_column>
df_smoking = pd.read_csv(".. /input/shareofadultswhosmoke/adults-smoking-2000-2016.csv")
 print(np.sort(df_smoking['Entity'].unique())[:10])
 df_smoking.head()
all_to_remove = fs.check_removal()<drop_column>
df_traintest3 = pd.merge(df_traintest2, df_smoking_recent[['Country/Region', 'SmokingRate']], left_on='Country_Region', right_on='Country/Region', how='left')
 df_traintest3.head()
train_removed_all = fs.remove(methods = 'all', keep_one_hot=False )<choose_model_class>
SmokingRate = df_smoking_recent['SmokingRate'][df_smoking_recent['Entity']=='World'].values[0]
print("Smoking rate of the world: {:.6f}".format(SmokingRate))
df_traintest3['SmokingRate'][pd.isna(df_traintest3['SmokingRate'])] = SmokingRate
df_traintest3.head()
def LGB_bayesian_sec(**lgb_bayes_params):
lgb_bayes_params['num_leaves'] = int(lgb_bayes_params['num_leaves'])
 lgb_bayes_params['max_depth'] = int(lgb_bayes_params['max_depth'])
 lgb_bayes_params['min_child_samples'] = int(lgb_bayes_params['min_child_samples'])
 lgb_bayes_params['max_bin'] = int(lgb_bayes_params['max_bin'])
 lgb_bayes_params['subsample_freq'] = int(lgb_bayes_params['subsample_freq'])
 lgb_bayes_params['min_child_weight'] = int(lgb_bayes_params['min_child_weight'])
 lgb_bayes_params['subsample_for_bin'] = int(lgb_bayes_params['subsample_for_bin'])
 lgb_bayes_params['n_estimators'] = int(lgb_bayes_params['n_estimators'])
 fix_lgb_params = {
'objective':'regression',
"boosting": "gbdt",
"metric": 'mae',
}
lgb_params = {**fix_lgb_params, **lgb_bayes_params}
oof_val, oof_predict, predictions, feature_importance_df, scores = run_cv_model(
 train_data=train_df,
train_target=train_y,
test_data=test_df,
model_fn=run_lgb,
params=lgb_params,
scoring=mean_absolute_error,
label="lgb",
feature_imp=True,
n_folds=8 
)
 return -np.mean(scores )<compute_train_metric>
world_happiness_index = pd.read_csv(".. /input/world-bank-datasets/World_Happiness_Index.csv")
 world_happiness_grouped = world_happiness_index.groupby('Country name' ).nth(-1)
 world_happiness_grouped.head()
world_happiness_grouped.drop("Year", axis=1, inplace=True)
 df_traintest3 = pd.merge(left=df_traintest3, right=world_happiness_grouped, how='left', left_on='Country_Region', right_on='Country name' )
oof_val, oof_predict, predictions, feature_importance_df, scores = run_cv_model(
 train_data=train_df,
train_target=train_y,
test_data=test_df,
model_fn=run_cat,
params=cat_params,
scoring=mean_absolute_error,
label="cat",
feature_imp=True,
n_folds=8
)<save_to_csv>
malaria_world_health = pd.read_csv(".. /input/world-bank-datasets/Malaria_World_Health_Organization.csv")
 df_traintest3 = pd.merge(left=df_traintest3, right=malaria_world_health, how='left', left_on='Country_Region', right_on='Country')
 df_traintest3.drop("Country", axis=1, inplace=True)
 mwh_cols = [ col for col in malaria_world_health.columns.to_list() if col != "Country" ]
print(mwh_cols )
submission.time_to_failure = predictions
submission.to_csv('clean_submission.csv',index=True )<load_from_csv>
human_development_index = pd.read_csv(".. /input/world-bank-datasets/Human_Development_Index.csv")
 human_development_index.drop(["Gross national income(GNI)per capita 2018"], axis=1, inplace=True)
 df_traintest3 = pd.merge(left=df_traintest3, right=human_development_index, how='left', left_on='Country_Region', right_on='Country')
 df_traintest3.drop("Country", axis=1, inplace=True)
 hdi_cols = [ col for col in human_development_index.columns.to_list() if col != "Country" ]
print(hdi_cols )
%%time
event_type=pd.read_csv(".. /input/event_type.csv",error_bad_lines=False)
 train = pd.read_csv(".. /input/train.csv")
 severity_type = pd.read_csv(".. /input/severity_type.csv")
 log_feature = pd.read_csv(".. /input/log_feature.csv")
 test = pd.read_csv(".. /input/test.csv")
 resource_type = pd.read_csv(".. /input/resource_type.csv",error_bad_lines=False)
 sample_submission = pd.read_csv(".. /input/sample_submission.csv" )<data_type_conversions>
event_type['id']=pd.to_numeric(event_type['id'],errors='coerce')
<merge>
train_eve_mer=train.merge(event_type.drop_duplicates(subset=['id']),on='id' )<merge>
df_lat_long = pd.read_csv(".. /input/lat-long/lat_long.csv")
 df_lat_long['place_id'] = df_lat_long.apply(lambda x: func(x), axis=1)
 df_lat_long.drop(["Country_Region", 'Province_State'], axis=1, inplace=True)
 df_traintest3 = pd.merge(left=df_traintest3, right=df_lat_long, how='left', on='place_id' )
train_merge2=train_eve_mer.merge(severity_type.drop_duplicates(subset=['id']),how='left',on='id' )<merge>
df_traintest3[df_traintest3['place_id']=='China/Hubei']
train_merge3=pd.merge(train_merge2,log_feature.drop_duplicates(subset=['id']),how='left',on='id' )<merge>
SEED = 42
params = {'num_leaves': 8,
'min_data_in_leaf': 5,
'objective': 'regression',
'max_depth': 8,
'learning_rate': 0.02,
'boosting': 'gbdt',
'bagging_freq': 5,
'bagging_fraction': 0.8,
'feature_fraction': 0.8201,
'bagging_seed': SEED,
'reg_alpha': 1,
'reg_lambda': 4.9847051755586085,
'random_state': SEED,
'metric': 'mse',
'verbosity': 100,
'min_gain_to_split': 0.02,
'min_child_weight': 5,
'num_threads': 6,
}
train_merge4=pd.merge(train_merge3,resource_type.drop_duplicates(subset=['id']),how='left',on='id' )<count_missing_values>
df_traintest3[df_traintest3.dayofyear == 72]
train_merge4.isnull().sum()<define_variables>
col_target = 'fatal/day'
col_var = [
'Lat', 'Long',
'cases/day_mean_(1-1)', 'cases/day_mean_(1-7)', 'cases/day_mean_(8-14)',
'fatal/day_mean_(1-1)', 'fatal/day_mean_(1-7)', 'fatal/day_mean_(8-14)',
'SmokingRate',
]
col_var += time_cols
extra_cols = wh_cols + mwh_cols + hdi_cols
col_var += extra_cols
df_train = df_traintest3[(pd.isna(df_traintest3['ForecastId'])) &(df_traintest3['dayofyear']<train_valid_cutoff_dayofyear)]
df_valid = df_traintest3[(pd.isna(df_traintest3['ForecastId'])) &(df_traintest3['dayofyear']>=train_valid_cutoff_dayofyear)]
df_test = df_traintest3[pd.isna(df_traintest3['ForecastId'])==False]
X_train = df_train[col_var].values
X_valid = df_valid[col_var].values
y_train = df_train[col_target].values
y_valid = df_valid[col_target].values
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 num_round = 15000
model = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=150, )
cat_col=list(set(train_merge4.columns)-set(train_merge4._get_numeric_data().columns))<data_type_conversions>
X_train = df_train[col_var].values
X_valid = df_valid[col_var].values
y_train = df_train[col_target].values
y_valid = df_valid[col_target].values
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 num_round = 15000
model = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=150, )
for i in range(len(cat_col)) :
train_merge4[cat_col[i]]=train_merge4[cat_col[i]].astype('category')
<categorify>
col_target2 = 'cases/day'
col_var2 = [
'Lat', 'Long',
'cases/day_mean_(1-1)', 'cases/day_mean_(1-7)', 'cases/day_mean_(8-14)',
'fatal/day_mean_(1-1)', 'fatal/day_mean_(1-7)', 'fatal/day_mean_(8-14)',
'SmokingRate',
]
col_var2 += time_cols
col_var2 += extra_cols
X_train = df_train[col_var2].values
X_valid = df_valid[col_var2].values
y_train = df_train[col_target2].values
y_valid = df_valid[col_target2].values
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 num_round = 15000
model2 = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=150, )
le=preprocessing.LabelEncoder()
for i in range(len(cat_col)) :
train_merge4[cat_col[i]]=le.fit_transform(train_merge4[cat_col[i]] )<drop_column>
tmp = pd.DataFrame()
tmp["feature"] = col_var2
tmp["importance"] = model2.feature_importance()
tmp = tmp.sort_values('importance', ascending=False)
 important_features = list(tmp[0:15]['feature'])
 col_var2 = important_features
tmp
train_merge4.drop(['id'],axis=1,inplace=True )<prepare_x_and_y>
X_train = df_train[col_var2].values
X_valid = df_valid[col_var2].values
y_train = df_train[col_target2].values
y_valid = df_valid[col_target2].values
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 num_round = 15000
model2 = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data],
verbose_eval=100,
early_stopping_rounds=150, )
target=train_merge4[['fault_severity']]<drop_column>
df_traintest4 = copy.deepcopy(df_traintest3)
 df_traintest4['unique'] = df_traintest4.apply(lambda x: x['place_id'] + str(x['dayofyear']), axis=1)
 print(len(df_traintest4))
df_traintest4 = df_traintest4[df_traintest4['unique'].duplicated() ==False]
print(len(df_traintest4))
df_traintest4[(df_traintest4['place_id']=='China/Hubei')&(df_traintest4['dayofyear']>75)].head()
train_merge4.drop(['fault_severity'],axis=1,inplace=True )<choose_model_class>
df_tmp = df_traintest[pd.isna(df_traintest['Fatalities'])==False]
df_tmp = df_tmp[df_tmp['dayofyear']<61]
df_agg = df_tmp.groupby('place_id')['Fatalities'].agg('max' ).reset_index()
df_agg = df_agg.sort_values('Fatalities', ascending=False)
 df_agg.head()
rfc=RandomForestClassifier()<train_model>
place = 'China/Hubei'
df_interest_base = df_traintest4[df_traintest4['place_id']==place].reset_index(drop=True)
 df_interest = copy.deepcopy(df_interest_base)
 df_interest['cases/day'] = df_interest['cases/day'].astype(np.float)
 df_interest['fatal/day'] = df_interest['fatal/day'].astype(np.float)
 df_interest['cases/day'][df_interest['dayofyear']>=train_valid_cutoff_dayofyear] = -1
df_interest['fatal/day'][df_interest['dayofyear']>=train_valid_cutoff_dayofyear] = -1
len_known =(df_interest['cases/day']!=-1 ).sum()
len_unknown =(df_interest['cases/day']==-1 ).sum()
print("len train: {}, len prediction: {}".format(len_known, len_unknown))
for i in range(len_unknown):
X_valid = df_interest[col_var].iloc[i+len_known]
X_valid2 = df_interest[col_var2].iloc[i+len_known]
pred_f = model.predict(X_valid)
 pred_c = model2.predict(X_valid2)
 df_interest['fatal/day'][i+len_known] = pred_f
df_interest['cases/day'][i+len_known] = pred_c
df_interest = df_interest[['cases/day', 'fatal/day', 'Long', 'Lat', 'SmokingRate']+time_cols+extra_cols]
df_interest = do_aggregations(df_interest)
 tmp = df_interest_base['fatal/day'].values
tmp = np.cumsum(tmp)
 sns.lineplot(x=df_interest_base['dayofyear'][pd.isna(df_interest_base['Fatalities'])==False],
y=tmp[pd.isna(df_interest_base['Fatalities'])==False], label='true')
 tmp = df_interest['fatal/day'].values
tmp = np.cumsum(tmp)
 sns.lineplot(x=df_interest_base['dayofyear'], y=tmp, label='pred')
 plt.show()
rfc.fit(train_merge4,target )<merge>
place = 'Iran'
df_interest_base = df_traintest4[df_traintest4['place_id']==place].reset_index(drop=True)
 df_interest = copy.deepcopy(df_interest_base)
 df_interest['cases/day'] = df_interest['cases/day'].astype(np.float)
 df_interest['fatal/day'] = df_interest['fatal/day'].astype(np.float)
 df_interest['cases/day'][df_interest['dayofyear']>=train_valid_cutoff_dayofyear] = -1
df_interest['fatal/day'][df_interest['dayofyear']>=train_valid_cutoff_dayofyear] = -1
len_known =(df_interest['cases/day']!=-1 ).sum()
len_unknown =(df_interest['cases/day']==-1 ).sum()
print("len train: {}, len prediction: {}".format(len_known, len_unknown))
for i in range(len_unknown):
X_valid = df_interest[col_var].iloc[i+len_known]
X_valid2 = df_interest[col_var2].iloc[i+len_known]
pred_f = model.predict(X_valid)
 pred_c = model2.predict(X_valid2)
 df_interest['fatal/day'][i+len_known] = pred_f
df_interest['cases/day'][i+len_known] = pred_c
df_interest = df_interest[['cases/day', 'fatal/day', 'Long', 'Lat', 'SmokingRate']+time_cols+extra_cols]
df_interest = do_aggregations(df_interest)
 tmp = df_interest_base['fatal/day'].values
tmp = np.cumsum(tmp)
 sns.lineplot(x=df_interest_base['dayofyear'][pd.isna(df_interest_base['Fatalities'])==False],
y=tmp[pd.isna(df_interest_base['Fatalities'])==False], label='true')
 tmp = df_interest['fatal/day'].values
tmp = np.cumsum(tmp)
 sns.lineplot(x=df_interest_base['dayofyear'], y=tmp, label='pred')
 plt.show()
test_merge1=pd.merge(test,event_type.drop_duplicates(subset=['id']),how='left',on='id' )<merge>
place = 'Italy'
df_interest_base = df_traintest4[df_traintest4['place_id']==place].reset_index(drop=True)
 df_interest = copy.deepcopy(df_interest_base)
 df_interest['cases/day'] = df_interest['cases/day'].astype(np.float)
 df_interest['fatal/day'] = df_interest['fatal/day'].astype(np.float)
 df_interest['cases/day'][df_interest['dayofyear']>=train_valid_cutoff_dayofyear] = -1
df_interest['fatal/day'][df_interest['dayofyear']>=train_valid_cutoff_dayofyear] = -1
len_known =(df_interest['cases/day']!=-1 ).sum()
len_unknown =(df_interest['cases/day']==-1 ).sum()
print("len train: {}, len prediction: {}".format(len_known, len_unknown))
for i in range(len_unknown):
X_valid = df_interest[col_var].iloc[i+len_known]
X_valid2 = df_interest[col_var2].iloc[i+len_known]
pred_f = model.predict(X_valid)
 pred_c = model2.predict(X_valid2)
 df_interest['fatal/day'][i+len_known] = pred_f
df_interest['cases/day'][i+len_known] = pred_c
df_interest = df_interest[['cases/day', 'fatal/day', 'Long', 'Lat', 'SmokingRate']+time_cols+extra_cols]
df_interest = do_aggregations(df_interest)
 tmp = df_interest_base['fatal/day'].values
tmp = np.cumsum(tmp)
 sns.lineplot(x=df_interest_base['dayofyear'][pd.isna(df_interest_base['Fatalities'])==False],
y=tmp[pd.isna(df_interest_base['Fatalities'])==False], label='true')
 tmp = df_interest['fatal/day'].values
tmp = np.cumsum(tmp)
 sns.lineplot(x=df_interest_base['dayofyear'], y=tmp, label='pred')
 plt.show()
test_merge2=pd.merge(test_merge1,severity_type.drop_duplicates(subset=['id']),how='left',on='id' )<merge>
df_train = df_traintest3[(pd.isna(df_traintest3['ForecastId'])) ]
X_train = df_train[col_var].values
X_valid = df_train[col_var].values
y_train = df_train[col_target].values
y_valid = df_train[col_target].values
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 num_round = 575
model = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data], verbose_eval=100, )
test_merge3=pd.merge(test_merge2,log_feature.drop_duplicates(subset=['id']),how='left',on='id' )<merge>
df_train = df_traintest3[(pd.isna(df_traintest3['ForecastId'])) ]
X_train = df_train[col_var2].values
X_valid = df_train[col_var2].values
y_train = df_train[col_target2].values
y_valid = df_train[col_target2].values
train_data = lgb.Dataset(X_train, label=y_train)
 valid_data = lgb.Dataset(X_valid, label=y_valid)
 num_round = 225
model2 = lgb.train(params, train_data, num_round, valid_sets=[train_data, valid_data], verbose_eval=100, )
test_merge4=pd.merge(test_merge3,resource_type.drop_duplicates(subset=['id']),how='left',on='id' )<data_type_conversions>
df_traintest4[(df_traintest4['place_id']=='China/Hubei')&(df_traintest4['dayofyear']>=72)]
for i in range(len(cat_col)) :
test_merge4[cat_col[i]]=test_merge4[cat_col[i]].astype('category')
<categorify>
df_preds = []
for i, place in enumerate(places[:]):
df_interest = copy.deepcopy(df_traintest4[df_traintest4['place_id']==place].reset_index(drop=True))
df_interest['cases/day'] = df_interest['cases/day'].astype(np.float)
 df_interest['fatal/day'] = df_interest['fatal/day'].astype(np.float)
 df_interest['cases/day'][(pd.isna(df_interest['ForecastId'])) ==False] = -1
df_interest['fatal/day'][(pd.isna(df_interest['ForecastId'])) ==False] = -1
len_known =(df_interest['cases/day']!=-1 ).sum()
len_unknown =(df_interest['cases/day']==-1 ).sum()
if(i+1)%10==0:
print("{:3d}/{} {}, len known: {}, len unknown: {}".format(i+1, len(places), place, len_known, len_unknown), df_interest.shape)
 for j in range(len_unknown):
X_valid = df_interest[col_var].iloc[j+len_known]
X_valid2 = df_interest[col_var2].iloc[j+len_known]
pred_f = model.predict(X_valid)
 pred_c = model2.predict(X_valid2)
 df_interest['fatal/day'][j+len_known] = pred_f
df_interest['cases/day'][j+len_known] = pred_c
df_interest = df_interest[['cases/day', 'fatal/day', 'Long', 'Lat', 'SmokingRate', 'ForecastId', 'place_id']+time_cols+extra_cols]
df_interest = do_aggregations(df_interest)
 df_interest['fatal_pred'] = np.cumsum(df_interest['fatal/day'].values)
 df_interest['cases_pred'] = np.cumsum(df_interest['cases/day'].values)
 df_preds.append(df_interest )
le2=preprocessing.LabelEncoder()
for i in range(len(cat_col)) :
test_merge4[cat_col[i]]=le2.fit_transform(test_merge4[cat_col[i]] )<drop_column>
df_preds= pd.concat(df_preds)
 df_preds = df_preds.sort_values('dayofyear')
 col_tmp = ['place_id', 'ForecastId', 'dayofyear', 'cases/day', 'cases_pred', 'fatal/day', 'fatal_pred',]
df_preds[col_tmp][(df_preds['place_id']=='Afghanistan')&(df_preds['dayofyear']>75)].head(10 )
test_merge4.drop(['id'],axis=1,inplace=True )<predict_on_test>
df_sub = pd.read_csv(".. /input/covid19-global-forecasting-week-2/submission.csv")
 print(len(df_sub))
df_sub.head()
predict_test=rfc.predict_proba(test_merge4 )<create_dataframe>
df_sub = pd.merge(df_sub, df_traintest3[['ForecastId', 'place_id', 'dayofyear']])
 df_sub = pd.merge(df_sub, df_preds[['place_id', 'dayofyear', 'cases_pred', 'fatal_pred']], on=['place_id', 'dayofyear',], how='left')
 df_sub.head(10 )
<concatenate><EOS> 
df_sub['ConfirmedCases'] = df_sub['cases_pred']
df_sub['Fatalities'] = df_sub['fatal_pred']
df_sub = df_sub[['ForecastId', 'ConfirmedCases', 'Fatalities']]
df_sub.to_csv("submission.csv", index=None)
 df_sub.head(10 )