Kaggle competition: COVID19 Global Forecasting (Week 2)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-2)<concatenate>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
train_text_features = hstack([train_text_es1, train_text_es2, train_text_prs, train_text_rsd, train_text_pjt] ).tocsr()
test_text_features = hstack([test_text_es1, test_text_es2, test_text_prs, test_text_rsd, test_text_pjt] ).tocsr()
train_text_features.shape, test_text_features.shape<drop_column>
EMPTY_VAL = "EMPTY_VAL"
NAN_VAL = "NaN"
def get_state(state, country):
if state == EMPTY_VAL: return country
if state == NAN_VAL: return country
return state
train_features = hstack([train_none_text_features, train_label_features, train_text_features] ).tocsr()
test_features = hstack([test_none_text_features, test_label_features, test_text_features] ).tocsr()
del train_text_es1
del train_text_es2
del train_text_prs
del train_text_rsd
del train_text_pjt
del test_text_es1
del test_text_es2
del test_text_prs
del test_text_rsd
del test_text_pjt
del train_text_features
del test_text_features
gc.collect()<train_model>
def calc_score(y_true, y_pred):
y_true[y_true<0] = 0
score = metrics.mean_squared_error(np.log(y_true.clip(0, 1e10)+1), np.log(y_pred[:]+1)) **0.5
return score
class qiaofeng_kfold_stack:
def __init__(self, train, train_target, test, split_target, model, preprocess_func=None, score_func=None, kfolds=5, random_seed=9527, logger=None):
self.train = train
self.train_target = train_target
self.test = test
self.split_target = split_target
self.model = model
self.preprocess_func = preprocess_func
self.score_func = score_func
self.kfolds = kfolds
self.random_seed = random_seed
self.logger= logger
self.skf = KFold(n_splits=self.kfolds, random_state= self.random_seed)
 self.predict_test_kfolds = []
self.predict_valid_kfolds = np.zeros(( self.train.shape[0]))
def print_params(self):
print('kfolds : %s' % self.kfolds)
 print('random seed : %s' % self.random_seed)
 def preprocess(self):
if self.preprocess_func != None:
self.train, self.test = self.preprocess_func(self.train, self.test)
 def score(self, target, predict):
return self.score_func(target, predict)
 def model_fit(self, train, train_target):
self.model.fit(train, train_target)
 def model_predict(self, dataset):
return self.model.predict(dataset)
 def model_fit_predict(self, train, train_target, dataset):
self.model_fit(train, train_target)
 predict_train = None
predict_valid = self.model_predict(dataset)
 predict_test = self.model_predict(self.test)
 return predict_train, predict_valid, predict_test
def clear_predicts(self):
self.predict_test_kfolds = []
self.predict_valid_kfolds = np.zeros(( self.train.shape[0]))
def model_train_with_kfold(self):
self.clear_predicts()
for(folder_index,(train_index, valid_index)) in enumerate(self.skf.split(self.train)) :
x_train, x_valid = self.train[train_index], self.train[valid_index]
y_train, y_valid = self.train_target[train_index], self.train_target[valid_index]
predict_train, predict_valid, predict_test = self.model_fit_predict(x_train, y_train, x_valid)
 self.predict_test_kfolds.append(predict_test)
 self.predict_valid_kfolds[valid_index] = predict_valid
if self.logger != None:
valid_score = self.score(y_valid, predict_valid)
 self.logger('Fold: %s, valid score: %s' %(folder_index, valid_score))
def predict_test_mean(self):
return reduce(lambda x,y:x+y, self.predict_test_kfolds)/ self.kfolds<predict_on_test>
PATH_WEEK2='/kaggle/input/covid19-global-forecasting-week-2'
df_train = pd.read_csv(f'{PATH_WEEK2}/train.csv')
 df_test = pd.read_csv(f'{PATH_WEEK2}/test.csv')
 df_train.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_test.rename(columns={'Country_Region':'Country'}, inplace=True)
 df_train.rename(columns={'Province_State':'State'}, inplace=True)
 df_test.rename(columns={'Province_State':'State'}, inplace=True)
 df_train['Date'] = pd.to_datetime(df_train['Date'], infer_datetime_format=True)
 df_test['Date'] = pd.to_datetime(df_test['Date'], infer_datetime_format=True)
 y1_Train = df_train.iloc[:, -2]
y2_Train = df_train.iloc[:, -1]
class qiaofeng_predict_prob(qiaofeng_kfold_stack):
def model_predict(self, dataset):
return self.model.predict_proba(dataset)[:,1]<choose_model_class>
X_Train = df_train.copy()
X_Train['State'].fillna(EMPTY_VAL, inplace=True)
 X_Train['State'] = X_Train.loc[:, ['State', 'Country']].apply(lambda x : get_state(x['State'], x['Country']), axis=1)
 X_Train.loc[:, 'Date'] = X_Train.Date.dt.strftime("%m%d")
 X_Train["Date"] = X_Train["Date"].astype(int)
 X_Train.head()
%%time
model = RandomForestClassifier(n_jobs=4,
criterion="entropy",
max_depth=20,
n_estimators=100,
max_features='sqrt',
random_state=233, 
)
 qiaofeng_rf = qiaofeng_predict_prob(train=train_features, train_target=train_target, test=test_features, kfolds=5,split_target=train_target,
score_func=roc_auc_score, logger=print, model=model)
 qiaofeng_rf.model_train_with_kfold()
pred_valid_rf = qiaofeng_rf.predict_valid_kfolds
pred_test_avg_rf = qiaofeng_rf.predict_test_mean()
del qiaofeng_rf
gc.collect()<train_model>
X_Pred = df_test.copy()
X_Pred['State'].fillna(EMPTY_VAL, inplace=True)
 X_Pred['State'] = X_Pred.loc[:, ['State', 'Country']].apply(lambda x : get_state(x['State'], x['Country']), axis=1)
 X_Pred.loc[:, 'Date'] = X_Pred.Date.dt.strftime("%m%d")
 X_Pred["Date"] = X_Pred["Date"].astype(int)
 X_Pred.head()
%%time
class qiaofeng_lgb_clf(qiaofeng_kfold_stack):
def model_fit(self, train, train_target):
X_tra, X_val, y_tra, y_val = train_test_split(train, train_target, train_size=0.95, random_state=233)
 self.model = lgb.LGBMClassifier(n_jobs=4,
max_depth=4,
metric="auc",
n_estimators=1000,
num_leaves=15,
boosting_type="gbdt",
learning_rate=0.05,
feature_fraction=0.45,
colsample_bytree=0.45,
bagging_fraction=0.8,
bagging_freq=5,
reg_lambda=0.2)
 self.model.fit(X=X_tra, y=y_tra,
eval_set=[(X_val, y_val)],
early_stopping_rounds=200,
verbose=False)
 def model_predict(self, dataset):
return self.model.predict_proba(dataset)[:,1]
qiaofeng_lgb = qiaofeng_lgb_clf(train=train_features, train_target=train_target, test=test_features, kfolds=5,split_target=train_target,
score_func=roc_auc_score, logger=print, model=None)
 qiaofeng_lgb.model_train_with_kfold()
pred_valid_lgb = qiaofeng_lgb.predict_valid_kfolds
pred_test_avg_lgb = qiaofeng_lgb.predict_test_mean()
del qiaofeng_lgb
gc.collect()<train_model>
le = preprocessing.LabelEncoder()
X_Train.Country = le.fit_transform(X_Train.Country)
 X_Train['State'] = le.fit_transform(X_Train['State'])
 X_Train.head()
%%time
class qiaofeng_lgb_clf(qiaofeng_kfold_stack):
def model_fit(self, train, train_target):
X_tra, X_val, y_tra, y_val = train_test_split(train, train_target, train_size=0.95, random_state=233)
 self.model = lgb.LGBMClassifier(n_jobs=4,
max_depth=4,
metric="auc",
n_estimators=1000,
num_leaves=15,
boosting_type="gbdt",
learning_rate=0.05,
feature_fraction=0.45,
colsample_bytree=0.45,
bagging_fraction=0.8,
bagging_freq=5,
reg_lambda=0.2)
 self.model.fit(X=X_tra, y=y_tra,
eval_set=[(X_val, y_val)],
early_stopping_rounds=200,
verbose=False)
 def model_predict(self, dataset):
return self.model.predict_proba(dataset)[:,1]
qiaofeng_lgb = qiaofeng_lgb_clf(train=train_features.astype(np.bool ).astype(np.float32),
train_target=train_target,
test=test_features.astype(np.bool ).astype(np.float32),
kfolds=5,split_target=train_target,
score_func=roc_auc_score,
logger=print, model=None)
 qiaofeng_lgb.model_train_with_kfold()
pred_valid_lgb_bool = qiaofeng_lgb.predict_valid_kfolds
pred_test_avg_lgb_bool = qiaofeng_lgb.predict_test_mean()
del qiaofeng_lgb
gc.collect()<compute_test_metric>
X_Pred.Country = le.fit_transform(X_Pred.Country)
 X_Pred['State'] = le.fit_transform(X_Pred['State'])
 X_Pred.head()
pd.Series(pred_valid_lgb ).corr(pd.Series(pred_valid_lgb_bool)) , pd.Series(pred_test_avg_lgb ).corr(pd.Series(pred_test_avg_lgb_bool))<split>
filterwarnings('ignore')
 le = preprocessing.LabelEncoder()
n_estimators = 5000
n_poly_degree = 5
countries = X_Train.Country.unique()
df_out_xgb = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 df_out_lgb = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 df_out_ply = pd.DataFrame({'ForecastId': [], 'ConfirmedCases': [], 'Fatalities': []})
 for country in countries:
states = X_Train.loc[X_Train.Country == country, :].State.unique()
for state in states:
X_Train_CS = X_Train.loc[(X_Train.Country == country)&(X_Train.State == state), ['State', 'Country', 'Date', 'ConfirmedCases', 'Fatalities']]
y_Train_CS_Cases = X_Train_CS.loc[:, 'ConfirmedCases']
y_Train_CS_Fatal = X_Train_CS.loc[:, 'Fatalities']
X_Train_CS = X_Train_CS.loc[:, ['State', 'Country', 'Date']]
X_Train_CS.Country = le.fit_transform(X_Train_CS.Country)
 X_Train_CS['State'] = le.fit_transform(X_Train_CS['State'])
 X_Pred_CS = X_Pred.loc[(X_Pred.Country == country)&(X_Pred.State == state), ['State', 'Country', 'Date', 'ForecastId']]
X_Pred_CS_Id = X_Pred_CS.loc[:, 'ForecastId']
X_Pred_CS = X_Pred_CS.loc[:, ['State', 'Country', 'Date']]
X_Pred_CS.Country = le.fit_transform(X_Pred_CS.Country)
 X_Pred_CS['State'] = le.fit_transform(X_Pred_CS['State'])
 model_xgb_cases = XGBRegressor(n_estimators = n_estimators)
 model_xgb_cases.fit(X_Train_CS, y_Train_CS_Cases)
 y_xgb_cases_pred = model_xgb_cases.predict(X_Pred_CS)
 model_xgb_fatal = XGBRegressor(n_estimators = n_estimators)
 model_xgb_fatal.fit(X_Train_CS, y_Train_CS_Fatal)
 y_xgb_fatal_pred = model_xgb_fatal.predict(X_Pred_CS)
 model_lgb_cases = XGBRegressor(n_estimators = n_estimators)
 model_lgb_cases.fit(X_Train_CS, y_Train_CS_Cases)
 y_lgb_cases_pred = model_lgb_cases.predict(X_Pred_CS)
 model_lgb_fatal = XGBRegressor(n_estimators = n_estimators)
 model_lgb_fatal.fit(X_Train_CS, y_Train_CS_Fatal)
 y_lgb_fatal_pred = model_lgb_fatal.predict(X_Pred_CS)
 pl_cases_pred = np.poly1d(np.polyfit(X_Train_CS.Date, y_Train_CS_Cases, n_poly_degree))
y_ply_cases_pred = pl_cases_pred(X_Pred_CS.Date)
 pl_fatal_pred = np.poly1d(np.polyfit(X_Train_CS.Date, y_Train_CS_Fatal, n_poly_degree))
y_ply_fatal_pred = pl_fatal_pred(X_Pred_CS.Date)
 df_xgb = pd.DataFrame({'ForecastId': X_Pred_CS_Id, 'ConfirmedCases': y_xgb_cases_pred, 'Fatalities': y_xgb_fatal_pred})
 df_lgb = pd.DataFrame({'ForecastId': X_Pred_CS_Id, 'ConfirmedCases': y_lgb_cases_pred, 'Fatalities': y_lgb_fatal_pred})
 df_ply = pd.DataFrame({'ForecastId': X_Pred_CS_Id, 'ConfirmedCases': y_ply_cases_pred, 'Fatalities': y_ply_fatal_pred})
 df_out_xgb = pd.concat([df_out_xgb, df_xgb], axis=0)
 df_out_lgb = pd.concat([df_out_lgb, df_lgb], axis=0)
 df_out_ply = pd.concat([df_out_ply, df_ply], axis=0)
clf = FTRL(alpha=0.01, beta=0.1, L1=0.1, L2=1000.0, D=train_features.shape[1], iters=1,
inv_link="identity", threads=4)
 X_tra, X_val, y_tra, y_val = train_test_split(train_features, train_target, train_size=0.95, random_state=7)
 train_scores = []
valid_scores = []
for i in range(10):
clf.fit(X_tra, y_tra)
 train_predict = clf.predict(X_tra)
 tp = train_predict - train_predict.min()
tp = tp / tp.max()
valid_predict = clf.predict(X_val)
 vp = valid_predict - valid_predict.min()
vp = vp / vp.max()
train_auc = roc_auc_score(y_tra, tp)
 valid_auc = roc_auc_score(y_val, vp)
 train_scores.append(train_auc)
 valid_scores.append(valid_auc)
 print(i+1, train_auc, valid_auc)
 plt.clf()
plt.figure(figsize=(10,10))
plt.plot(np.arange(len(train_scores)) , train_scores, label='train')
 plt.plot(np.arange(len(valid_scores)) , valid_scores, label='valid')
 valid_max = max(valid_scores)
 plt.title('%s - %s'%(valid_scores.index(valid_max)+1, valid_max))
plt.legend()
plt.show()<train_model>
df_out_xgb.ForecastId = df_out_xgb.ForecastId.astype('int')
 df_out_lgb.ForecastId = df_out_lgb.ForecastId.astype('int')
 df_out_ply.ForecastId = df_out_ply.ForecastId.astype('int' )
%%time
def sigmoid(x):
return 1 /(1 + np.exp(-x))
class qiaofeng_ftrl(qiaofeng_kfold_stack):
def model_fit(self, train, train_target):
self.model = FTRL(alpha=0.01, beta=0.1, L1=0.1, L2=1000.0, D=train_features.shape[1], iters=2,
inv_link="identity", threads=4)
 self.model.fit(train, train_target)
 def model_predict(self, dataset):
predict = self.model.predict(dataset)
 pred_nan = np.isnan(predict)
 if pred_nan.shape[0] == predict.shape[0]:
predict[pred_nan] = 0
else:
predict[pred_nan] = np.nanmean(predict)
 return sigmoid(predict)
 qiaofeng_ftrl = qiaofeng_ftrl(train=train_features, train_target=train_target, test=test_features, kfolds=20, split_target=train_target,
score_func=roc_auc_score, logger=print, model=None)
 qiaofeng_ftrl.model_train_with_kfold()
pred_valid_ftrl = qiaofeng_ftrl.predict_valid_kfolds
pred_test_avg_ftrl = qiaofeng_ftrl.predict_test_mean()<set_options>
df_out = df_out_ply.copy()
del train_features
del test_features
gc.collect()<string_transform>
df_out['ConfirmedCases'] =(1/4)*(df_out_xgb['ConfirmedCases'] + df_out_lgb['ConfirmedCases'])+(1/2)* df_out_ply['ConfirmedCases']
df_out['Fatalities'] =(1/4)*(df_out_xgb['Fatalities'] + df_out_lgb['Fatalities'])+(1/2)* df_out_ply['Fatalities']
tokenizer = text.Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_df['project_descp'])+ list(test_df['project_descp'])+ list(train_df['res_description'])+ list(test_df['res_description']))
train_pj = sequence.pad_sequences(tokenizer.texts_to_sequences(train_df['project_descp']), maxlen=project_maxlen)
 test_pj = sequence.pad_sequences(tokenizer.texts_to_sequences(test_df['project_descp']), maxlen=project_maxlen)
 train_res = sequence.pad_sequences(tokenizer.texts_to_sequences(train_df['res_description']), maxlen=resouse_max_len)
 test_res = sequence.pad_sequences(tokenizer.texts_to_sequences(test_df['res_description']), maxlen=resouse_max_len)
 train_seq = np.hstack([train_pj, train_res])
 test_seq = np.hstack([test_pj, test_res] )<feature_engineering>
df_out['ConfirmedCases'] = df_out['ConfirmedCases'].round().astype(int)
 df_out['Fatalities'] = df_out['Fatalities'].round().astype(int )
<drop_column><EOS> 
df_out.to_csv('submission.csv', index=False )