Kaggle competition: COVID19 Global Forecasting (Week 4)
FIRST_DAY = 400<correct_missing_values>
operation_mode = 'final';
df.dropna(inplace = True)
 df.shape<prepare_x_and_y>
from sklearn.base import TransformerMixin, BaseEstimator
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import PolynomialFeatures
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_log_error
cat_feats = ['item_id', 'dept_id','store_id', 'cat_id', 'state_id'] + ["event_name_1", "event_name_2", "event_type_1", "event_type_2"]
useless_cols = ["id", "date", "sales","d", "wm_yr_wk", "weekday"]
train_cols = df.columns[~df.columns.isin(useless_cols)]
X_train = df[train_cols]
y_train = df["sales"]<create_dataframe>
df_train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv');
df_test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv');
df_train.shape, df_test.shape
<create_dataframe>
df_train['Date'] = pd.to_datetime(df_train['Date'], format = '%Y-%m-%d');
df_test['Date'] = pd.to_datetime(df_test['Date'], format = '%Y-%m-%d');
%%time
np.random.seed(777)
 train_inds = X_train.index.values
train_data = lgb.Dataset(X_train.loc[train_inds] , label = y_train.loc[train_inds], categorical_feature=cat_feats, free_raw_data=False)
<prepare_x_and_y>
def combine_country_province(df):
df.loc[:,'Province_State'] = df['Province_State'].fillna("")
 df.loc[:,'Region'] = df['Country_Region'] + " " + df['Province_State']
df.loc[:,'Region'] = df.loc[:,'Region'].str.strip() ;
return df;
df_train = combine_country_province(df_train);
df_test = combine_country_province(df_test);
df_train
del df, X_train, y_train ; gc.collect()<init_hyperparams>
class Days_Since_P0_World(BaseEstimator, TransformerMixin):

 def __init__(self):
self.p_zero_date = None;
self.col_name = 'days_since_p0_world';
pass
def fit(self, X, y=None):
self.p_zero_date = X['Date'].min()
return self;
def transform(self, X):
X[self.col_name] = X['Date'] - self.p_zero_date;
X.loc[:,self.col_name] = X[self.col_name].dt.days;
return X;
params = {
"objective" : "poisson",
"metric" :"rmse",
"force_row_wise" : True,
"learning_rate" : 0.08,
"sub_row" : 0.75,
"bagging_freq" : 1,
"lambda_l2" : 0.1,
"metric": ["rmse"],
'verbosity': 1,
'num_iterations' : 1200,
'num_leaves': 128,
"min_data_in_leaf": 100,
}<train_model>
days_since_p0_world = Days_Since_P0_World()
df_train = days_since_p0_world.fit_transform(df_train)
 df_test = days_since_p0_world.transform(df_test)
 df_train
<define_variables>
class Days_Since_P0_Country(BaseEstimator, TransformerMixin):

 def __init__(self, y_col_name = 'ConfirmedCases'):
self.p_zero_date = {};
self.col_name = 'days_since_p0_country';
self.y_col_name = y_col_name;
pass
def fit(self, X, y=None):
regions = X['Country_Region'].unique() ;
for this_region in regions:
this_region_X = X.loc[X['Country_Region'] == this_region,:];
self.p_zero_date[this_region] = min(this_region_X.loc[this_region_X[self.y_col_name]>0,'Date']);
return self;
def transform(self, X):
regions = X['Country_Region'].unique() ;
X[self.col_name] = 0;
answer = pd.DataFrame() ;
for this_region in regions:
this_region_X = None;
this_region_X = X.loc[X['Country_Region'] == this_region,:];
this_region_X.loc[:,self.col_name] = this_region_X['Date'] - self.p_zero_date[this_region];
this_region_X.loc[:,self.col_name] = this_region_X[self.col_name].dt.days;
answer = pd.concat([answer, this_region_X], axis='index');
return answer;
m_lgb = lgb.Booster(model_file='/kaggle/input/m5-accuracy-submission-file/model.lgb' )<save_model>
days_since_p0_country = Days_Since_P0_Country()
df_train = days_since_p0_country.fit_transform(df_train)
 df_test = days_since_p0_country.transform(df_test)
 df_train
m_lgb.save_model("model.lgb" )<define_variables>
class Days_Since_P0_Region(BaseEstimator, TransformerMixin):

 def __init__(self, y_col_name = 'ConfirmedCases'):
self.p_zero_date = {};
self.col_name = 'days_since_p0_region';
self.y_col_name = y_col_name;
pass
def fit(self, X, y=None):
regions = X['Region'].unique() ;
for this_region in regions:
this_region_X = X.loc[X['Region'] == this_region,:];
self.p_zero_date[this_region] = min(this_region_X.loc[this_region_X[self.y_col_name]>0,'Date']);
return self;
def transform(self, X):
regions = X['Region'].unique() ;
X[self.col_name] = 0;
answer = pd.DataFrame() ;
for this_region in regions:
this_region_X = None;
this_region_X = X.loc[X['Region'] == this_region,:];
this_region_X.loc[:,self.col_name] = this_region_X['Date'] - self.p_zero_date[this_region];
this_region_X.loc[:,self.col_name] = this_region_X[self.col_name].dt.days;
answer = pd.concat([answer, this_region_X], axis='index');
return answer;
fday = datetime(2016,5,23)
 fday<save_to_csv>
days_since_p0_region = Days_Since_P0_Region()
df_train = days_since_p0_region.fit_transform(df_train)
 df_test = days_since_p0_region.transform(df_test)
 df_train
sub = pd.concat([sub2, sub], axis=0, sort=False)
 sub.to_csv("submission.csv",index=False )<count_unique_values>
df_train.loc[df_train['Province_State'] == 'Alabama',:]
df_train.loc[(df_train['Date'] == '2020-01-22')&(df_train['Country_Region'] == 'China'),:]
df_train.loc[(df_train['Date'] == '2020-01-23')&(df_train['Country_Region'] == 'China'),:]
sub.id.nunique() , sub["id"].str.contains("validation$" ).sum()<count_unique_values>
def rmsle(y_true, y_pred):
return mean_squared_log_error(y_true, y_pred)**(1/2);
sub.id.nunique() , sub["id"].str.contains("evaluation$" ).sum()<set_options>
degree = 3
poly = PolynomialFeatures(degree = degree, include_bias=False)
 model1 = LinearRegression()
model2 = LinearRegression()
pd.set_option('display.max_columns', 500)
 pd.set_option('display.max_rows', 500)
 warnings.filterwarnings('ignore')
<define_variables>
X_cols = ['days_since_p0_region','days_since_p0_country','days_since_p0_world'];
y1_col = ['ConfirmedCases']
y2_col = ['Fatalities']
all_pred_train = pd.DataFrame() ;
all_pred_test = pd.DataFrame() ;
regions = df_train['Region'].unique() ;
if operation_mode == 'validation':
train_test_split_date = '2020-04-01';
train = df_train.loc[(df_train['Date'] < train_test_split_date),:];
test = df_train.loc[~(df_train['Date'] < train_test_split_date),:];
elif operation_mode == 'final':
train = df_train.copy() ;
test = df_test.copy() ;
INPUT_DIR_PATH = '.. /input/mlip-daemencloudt-feature-computation-notebook/'<split>
for idx, region in enumerate(regions):
scaler = StandardScaler() ;
this_region_train = train['Region'] == region;
this_region_test = test['Region'] == region;
X0_train_iter = train.loc[this_region_train,X_cols];
y1_train_iter = train.loc[this_region_train,y1_col];
y2_train_iter = train.loc[this_region_train,y2_col];
X0_test_iter = test.loc[this_region_test,X_cols];
X0_train_iter = poly.fit_transform(X0_train_iter);
X0_test_iter = poly.fit_transform(X0_test_iter);
X0_train_iter = scaler.fit_transform(X0_train_iter)
 X0_test_iter = scaler.transform(X0_test_iter)
 model1.fit(X0_train_iter, y1_train_iter);
y1_train_iter_pred = model1.predict(X0_train_iter);
y1_test_iter_pred = model1.predict(X0_test_iter);
model2.fit(X0_train_iter, y2_train_iter);
y2_train_iter_pred = model2.predict(X0_train_iter);
y2_test_iter_pred = model2.predict(X0_test_iter);
pred_iter_train = pd.DataFrame({
'Id': train.loc[this_region_train,'Id'],
'ConfirmedCases': y1_train_iter_pred.reshape(-1),
'Fatalities': y2_train_iter_pred.reshape(-1)
 })
 all_pred_train = pd.concat([all_pred_train, pred_iter_train], axis = 0);
if(operation_mode == 'validation'):
pred_iter_test = pd.DataFrame({
'Id': test.loc[this_region_test,'Id'],
'ConfirmedCases': y1_test_iter_pred.reshape(-1),
'Fatalities': y2_test_iter_pred.reshape(-1)
 })
 elif operation_mode == 'final':
pred_iter_test = pd.DataFrame({
'ForecastId': test.loc[this_region_test,'ForecastId'],
'ConfirmedCases': y1_test_iter_pred.reshape(-1),
'Fatalities': y2_test_iter_pred.reshape(-1)
 })
 all_pred_test = pd.concat([all_pred_test, pred_iter_test], axis = 0);
print(all_pred_train)
 print(all_pred_test )
def train_test_split(data):
x_train = data[data['date'] <= '2016-03-27']
y_train = x_train['demand']
x_val = data[(data['date'] > '2016-03-27')&(data['date'] <= '2016-04-24')]
y_val = x_val['demand']
test = data[(data['date'] > '2016-04-24')]
del data
gc.collect()
return x_train, y_train, x_val, y_val, test<train_model>
all_pred_test = all_pred_test.astype('int')
 all_pred_test.to_csv("submission.csv", index = False);
all_pred_test
def run_lgb(x_train, y_train, x_val, y_val, test):
paramstweedie= {
'boosting_type': 'gbdt',
'metric': 'custom',
'objective': 'tweedie',
'tweedie_variance_power': 1.1,
'n_jobs': -1,
'subsample': 0.5,
'subsample_freq': 1,
'learning_rate': 0.03,
'num_leaves': 2**11-1,
'min_data_in_leaf': 2**12-1,
'feature_fraction': 0.5,
'max_bin': 100,
'n_estimators': 1400,
'num_iterations' : 2000,
'boost_from_average': False,
'verbose': -1,
}
params = {
'metric': 'custom',
'objective': 'tweedie',
'tweedie_variance_power': 1.1,
'seed': 20,
'learning_rate': 0.1,
'num_iterations':2000,
'max_bin':500,
'feature fraction': 0.6,
'num_leaves': 100,
}
evals_result = {}
train_set = lgb.Dataset(x_train[features], y_train)
 val_set = lgb.Dataset(x_val[features], y_val)
 del x_train, y_train
model = lgb.train(params, train_set,early_stopping_rounds = 250, valid_sets = [train_set, val_set], verbose_eval = 10, feval= wrmsse,\
evals_result=evals_result)
 ax = lgb.plot_metric(evals_result)
 plt.show()
val_pred = model.predict(x_val[features], num_iteration=model.best_iteration)
 y_pred = model.predict(test[features], num_iteration=model.best_iteration)
 test['demand'] = y_pred
del y_pred
gc.collect()
return test, model<compute_test_metric>
def rmse(predictions, y):
return np.sqrt(mean_squared_error(y, predictions))<save_to_csv>
if operation_mode == 'validation':
answer = pd.merge(df_train,all_pred_test, left_on = 'Id',right_on = 'Id');
if operation_mode == 'final':
answer = pd.merge(df_train,all_pred_test, left_on = 'Id',right_on = 'ForecastId');
answer.loc[answer['ConfirmedCases_y'] < 0,:] = 0;
answer.loc[answer['Fatalities_y'] < 0,:] = 0;
print(rmsle(answer['ConfirmedCases_x'],answer['ConfirmedCases_y']))
print(rmsle(answer['Fatalities_x'],answer['Fatalities_y']))