Kaggle competition: COVID19 Global Forecasting (Week 1)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-1)<save_to_csv>
warnings.filterwarnings("ignore")
column_name = ['lgb_' + str(i)for i in range(9)]
pd.DataFrame(oof, columns = column_name ).to_csv('oof_lgb.csv', index=False)
 pd.DataFrame(y_pred_test, columns = column_name ).to_csv('submit_lgb.csv', index=False )<load_from_csv>
datapath = '.. /input/covid19-global-forecasting-week-1/'
train = pd.read_csv(datapath+'train.csv')
 test = pd.read_csv(datapath+'test.csv' )
class PreProcess:
def __init__(self):
self.df_otto_train = pd.read_csv('/kaggle/input/otto-group-product-classification-challenge/train.csv')
 self.df_otto_test = pd.read_csv('/kaggle/input/otto-group-product-classification-challenge/test.csv')
 def __call__(self):
conv = lambda x: int(x.replace('Class_', '')) - 1
self.df_otto_train['target2'] = self.df_otto_train.target.map(conv)
 x = self.df_otto_train.drop(['id', 'target', 'target2'], axis=1 ).values
y = self.df_otto_train.target2.values
x_test_id = self.df_otto_test.id.values
x_test = self.df_otto_test.drop(['id'], axis=1 ).values
x_train, x_valid, t_train, t_valid = train_test_split(x, y, test_size=0.2, shuffle=True)
 _, _, _, counts = np.unique(t_train, return_index=True, return_inverse=True, return_counts=True)
 loss_weight =(1 / torch.Tensor(counts)) *(x_train.shape[0]/len(np.unique(t_train)))
 loss_weight = loss_weight.to(device)
 return x_train, x_valid, t_train, t_valid, x_test, x_test_id, loss_weight<choose_model_class>
train['Date'] = train['Date'].astype('datetime64[ns]')
 test['Date'] = test['Date'].astype('datetime64[ns]')
 print("Train Date type: ", train['Date'].dtype)
 print("Test Date type: ",test['Date'].dtype )
class MLP(nn.Module):
def __init__(self, in_dim, out_dim):
super(MLP, self ).__init__()
n_layers = 5
layers = []
features = [522, 1006, 1304, 706, 851]
p = [0.1287716033668473, 0.08497812991732616, 0.19230277862176845, 0.09567140996248122, 0.15776969001259095]
in_features = in_dim
for i in range(n_layers):
out_features = features[i]
layers.append(nn.Linear(in_features, out_features))
layers.append(nn.BatchNorm1d(out_features))
layers.append(nn.ReLU())
 layers.append(nn.Dropout(p[i]))
in_features = out_features
layers.append(nn.Linear(in_features, out_dim))
layers.append(nn.LogSoftmax(dim=-1))
self.network = nn.Sequential(*layers)
 def forward(self, x):
return self.network(x)
 class MLP_tune(nn.Module):
def __init__(self, trial, in_dim, out_dim):
super(MLP_tune, self ).__init__()
n_layers = trial.suggest_int("n_layers", 5, 10)
 layers = []
in_features = in_dim
for i in range(n_layers):
out_features = trial.suggest_int("n_units_l{}".format(i), in_dim, in_dim*20)
 layers.append(nn.Linear(in_features, out_features))
layers.append(nn.BatchNorm1d(out_features))
layers.append(nn.ReLU())
 p = trial.suggest_float("dropout_l{}".format(i), 0.0, 0.5)
 layers.append(nn.Dropout(p))
in_features = out_features
layers.append(nn.Linear(in_features, out_dim))
layers.append(nn.LogSoftmax(dim=-1))
self.network = nn.Sequential(*layers)
 def forward(self, x):
return self.network(x )<train_model>
train.columns = ['id','state','country','lat','lon','date','ConfirmedCases','Fatalities']
test.columns = ['ForecastId', 'state','country','lat','lon','date']
class Train:
def __init__(self):
self.x_train, self.x_valid, self.t_train, self.t_valid, self.x_test, self.x_test_id, self.loss_weight = PreProcess()()
def train(self, n_epochs, lr, weight_decay):
in_dim = self.x_train.shape[1]
out_dim = len(np.unique(self.t_train))
mlp = MLP(in_dim, out_dim ).to(device)
 optimizer = optim.RMSprop(mlp.parameters() , lr=lr, weight_decay=weight_decay)
 scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')
 criterion = nn.NLLLoss(weight=self.loss_weight)
 ds_train = TensorDataset(torch.tensor(self.x_train ).float() , torch.tensor(self.t_train))
ds_valid = TensorDataset(torch.tensor(self.x_valid ).float() , torch.tensor(self.t_valid))
self.ds_test = TensorDataset(torch.tensor(self.x_test ).float())
 loader_train = DataLoader(ds_train, batch_size=256, shuffle=True)
 loader_valid = DataLoader(ds_valid, batch_size=256, shuffle=True)
 start_time = time.time()
for epoch in range(n_epochs):
losses_train = []
losses_valid = []
preds_train = []
preds_valid = []
trues_train = []
trues_valid = []
mlp.train()
for x, t in loader_train:
true = t.tolist()
trues_train.extend(true)
 mlp.zero_grad()
x = x.to(device)
 t = t.to(device)
 y = mlp.forward(x)
 loss = criterion(y, t.long())
 loss.backward()
optimizer.step()
pred = y.argmax(1 ).tolist()
preds_train.extend(pred)
 losses_train.append(loss.tolist())
 mlp.eval()
for x, t in loader_valid:
true = t.tolist()
trues_valid.extend(true)
 x = x.to(device)
 t = t.to(device)
 y = mlp.forward(x)
 loss = criterion(y, t.long())
 pred = y.argmax(1 ).tolist()
preds_valid.extend(pred)
 losses_valid.append(loss.tolist())
 print('EPOCH: {}, Train [Loss: {:.3f}, F1: {:.3f}, R: {:.3f}, P: {:.3f}], Valid [Loss: {:.3f}, F1: {:.3f}, R: {:.3f}, P: {:.3f}]'.format(
 epoch,
np.mean(losses_train),
f1_score(trues_train, preds_train, average='weighted'),
recall_score(trues_train, preds_train, average='weighted'),
precision_score(trues_train, preds_train, average='weighted'),
np.mean(losses_valid),
f1_score(trues_valid, preds_valid, average='weighted'),
recall_score(trues_valid, preds_valid, average='weighted'),
precision_score(trues_valid, preds_valid, average='weighted')
))
scheduler.step(np.mean(losses_valid))
elapsed_time = time.time() - start_time
print("elapsed_time:{:.1f}".format(elapsed_time / 60)+ "[min]")
 return mlp
def test(self, mlp):
loader_test = DataLoader(self.ds_test, batch_size=256, shuffle=False)
 preds_test = []
mlp.eval()
for x in loader_test:
x = x[0].to(device)
 y = mlp.forward(x)
 y = y.to('cpu' ).detach().numpy()
y = np.exp(y)
 preds_test.extend(y)
 return preds_test
def submit(self, preds_test):
df_sample_submission = pd.read_csv('/kaggle/input/otto-group-product-classification-challenge/sampleSubmission.csv')
 df_submission = pd.DataFrame(columns = df_sample_submission.columns)
 df_submission.id = self.x_test_id
df_submission[df_submission.columns[1:]] = preds_test
df_submission.to_csv('submission.csv', index=False)
 return
def get_optimizer(self, trial, model):
optimizer_names = ['Adam', 'MomentumSGD', 'rmsprop']
optimizer_name = trial.suggest_categorical('optimizer', optimizer_names)
 weight_decay = trial.suggest_loguniform('weight_decay', 1e-10, 1e-3)
 if optimizer_name == optimizer_names[0]:
adam_lr = trial.suggest_loguniform('adam_lr', 1e-5, 1e-1)
 optimizer = optim.Adam(model.parameters() , lr=adam_lr, weight_decay=weight_decay)
 elif optimizer_name == optimizer_names[1]:
momentum_sgd_lr = trial.suggest_loguniform('momentum_sgd_lr', 1e-5, 1e-1)
 optimizer = optim.SGD(model.parameters() , lr=momentum_sgd_lr, momentum=0.9, weight_decay=weight_decay)
 else:
rmsprop_lr = trial.suggest_loguniform('rmsprop_lr', 1e-5, 1e-1)
 optimizer = optim.RMSprop(model.parameters() , lr=rmsprop_lr, weight_decay=weight_decay)
 return optimizer
def objective(self, trial):
self.x_train, self.x_valid, self.t_train, self.t_valid, self.x_test, self.x_test_id, self.loss_weight = PreProcess()()
in_dim = self.x_train.shape[1]
out_dim = len(np.unique(self.t_train))
n_epochs = 100
mlp = MLP_tune(trial, in_dim, out_dim ).to(device)
 optimizer = self.get_optimizer(trial, mlp)
 scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')
 criterion = nn.NLLLoss(weight=self.loss_weight)
 ds_train = TensorDataset(torch.tensor(self.x_train ).float() , torch.tensor(self.t_train))
ds_valid = TensorDataset(torch.tensor(self.x_valid ).float() , torch.tensor(self.t_valid))
loader_train = DataLoader(ds_train, batch_size=256, shuffle=True)
 loader_valid = DataLoader(ds_valid, batch_size=256, shuffle=True)
 start_time = time.time()
for epoch in range(n_epochs):
losses_train = []
losses_valid = []
preds_train = []
preds_valid = []
trues_train = []
trues_valid = []
mlp.train()
for x, t in loader_train:
true = t.tolist()
trues_train.extend(true)
 mlp.zero_grad()
x = x.to(device)
 t = t.to(device)
 y = mlp.forward(x)
 loss = criterion(y, t.long())
 loss.backward()
optimizer.step()
pred = y.argmax(1 ).tolist()
preds_train.extend(pred)
 losses_train.append(loss.tolist())
 mlp.eval()
for x, t in loader_valid:
true = t.tolist()
trues_valid.extend(true)
 x = x.to(device)
 t = t.to(device)
 y = mlp.forward(x)
 loss = criterion(y, t.long())
 pred = y.argmax(1 ).tolist()
preds_valid.extend(pred)
 losses_valid.append(loss.tolist())
 print('EPOCH: {}, Train [Loss: {:.3f}, F1: {:.3f}, R: {:.3f}, P: {:.3f}], Valid [Loss: {:.3f}, F1: {:.3f}, R: {:.3f}, P: {:.3f}]'.format(
 epoch,
np.mean(losses_train),
f1_score(trues_train, preds_train, average='weighted'),
recall_score(trues_train, preds_train, average='weighted'),
precision_score(trues_train, preds_train, average='weighted'),
np.mean(losses_valid),
f1_score(trues_valid, preds_valid, average='weighted'),
recall_score(trues_valid, preds_valid, average='weighted'),
precision_score(trues_valid, preds_valid, average='weighted')
))
scheduler.step(np.mean(losses_valid))
metric = np.mean(losses_valid)
 trial.report(metric, epoch)
 if trial.should_prune() :
raise optuna.exceptions.TrialPruned()
elapsed_time = time.time() - start_time
print("elapsed_time:{:.1f}".format(elapsed_time / 60)+ "[min]")
 return metric<train_model>
train['place'] = train['state'].fillna('')+ '_' + train['country']
test['place'] = test['state'].fillna('')+ '_' + test['country']
def run_mlp(n_cross_valid):
n_cross_valid = n_cross_valid
preds_test_all = []
for n in range(n_cross_valid):
n_epochs = 50
lr = 7.560243948481283e-05
weight_decay = 0.0005896154168582327
train = Train()
mlp = train.train(n_epochs, lr, weight_decay)
 preds_test = train.test(mlp)
 preds_test_all.append(preds_test)
 preds_test = np.mean(preds_test_all, axis=0)
 return preds_test<train_model>
print('How many places?: ', 'Train: ', len(train['place'].unique()),
'Test: ', len(test['place'].unique()))
print('Unique place similar as test?: ',(train['place'].unique() == test['place'].unique() ).sum() )
def tune() :
train = Train()
study = optuna.create_study(direction="minimize")
 study.optimize(train.objective, n_trials=1000)
 pruned_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.PRUNED]
complete_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.COMPLETE]
print("Study statistics: ")
 print(" Number of finished trials: ", len(study.trials))
print(" Number of pruned trials: ", len(pruned_trials))
print(" Number of complete trials: ", len(complete_trials))
print("Best trial:")
 trial = study.best_trial
print(" Value: ", trial.value)
 print(" Params: ")
 for key, value in trial.params.items() :
print(" {}: {}".format(key, value)) 
<create_dataframe>
china_cases = train[train['place'].str.contains('China')][['date',
'ConfirmedCases',
'Fatalities']].reset_index(drop=True)
 restworld_cases = train[-train['place'].str.contains('China')][['date',
'ConfirmedCases',
'Fatalities']].reset_index(drop=True )
def run_lgb(n_cross_valid):
n_cross_valid = n_cross_valid
preds_test_all = []
for n in range(n_cross_valid):
x_train, x_valid, t_train, t_valid, x_test, x_test_id, _ = PreProcess()()
train_data = lgb.Dataset(x_train, label=t_train)
 eval_data = lgb.Dataset(x_valid, label=t_valid, reference=train_data)
 params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'multiclass',
"lambda_l1": 3.234625291884511e-05,
"lambda_l2": 0.0001436287400101119,
'num_leaves': 180,
"feature_fraction": 0.8978571608212189,
"bagging_fraction": 0.48465224832769926,
"bagging_freq": 1,
"min_child_samples": 15,
'learning_rate': 0.024535585298549177,
'num_class': 9,
'verbose': -1,
'device_type': 'gpu'
}
gbm = lgb.train(
 params,
train_data,
valid_sets=eval_data,
num_boost_round = 2000,
verbose_eval = 5,
early_stopping_rounds = 5 
)
 preds_test = gbm.predict(x_test)
 preds_test_all.append(preds_test)
 preds_test = np.mean(preds_test_all, axis=0)
 return preds_test<split>
us_cases = train[train['place'].str.contains('US')][['date','place',
'ConfirmedCases',
'Fatalities']].reset_index(drop=True )
def lgb_objective(trial):
x_train, x_valid, t_train, t_valid, _, _, _ = PreProcess()()
train_data = lgb.Dataset(x_train, label=t_train)
 eval_data = lgb.Dataset(x_valid, label=t_valid, reference=train_data)
 params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'multiclass',
"lambda_l1": trial.suggest_float("lambda_l1", 1e-8, 10.0, log=True),
"lambda_l2": trial.suggest_float("lambda_l2", 1e-8, 10.0, log=True),
'num_leaves': trial.suggest_int("num_leaves", 2, 256),
"feature_fraction": trial.suggest_float("feature_fraction", 0.4, 1.0),
"bagging_fraction": trial.suggest_float("bagging_fraction", 0.4, 1.0),
"bagging_freq": trial.suggest_int("bagging_freq", 1, 7),
"min_child_samples": trial.suggest_int("min_child_samples", 5, 100),
'learning_rate': trial.suggest_float("learning_rate", 0.001, 1.0, log=True),
'num_class': 9,
'verbose': -1,
}
pruning_callback = optuna.integration.LightGBMPruningCallback(trial, "multi_logloss")
 gbm = lgb.train(
 params,
train_data,
valid_sets = eval_data,
num_boost_round = 2000,
verbose_eval = 5,
early_stopping_rounds = 5,
callbacks=[pruning_callback] 
)
 return gbm.best_score['valid_0']['multi_logloss']
def tune_lgb() :
pruner = optuna.pruners.MedianPruner(n_startup_trials=20, n_warmup_steps=20)
 study = optuna.create_study(direction="minimize", pruner = pruner)
 study.optimize(lgb_objective, n_trials=300)
 pruned_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.PRUNED]
complete_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.COMPLETE]
print("Study statistics: ")
 print(" Number of finished trials: ", len(study.trials))
print(" Number of pruned trials: ", len(pruned_trials))
print(" Number of complete trials: ", len(complete_trials))
print("Best trial:")
 trial = study.best_trial
print(" Value: {}".format(trial.value))
print(" Params: ")
 for key, value in trial.params.items() :
print(" {}: {}".format(key, value)) 
<prepare_x_and_y>
def RMSLE(predicted, actual):
return np.sqrt(np.mean(np.power(( np.log(predicted+1)-np.log(actual+1)) ,2)) )
def run_xgb(n_cross_valid):
n_cross_valid = n_cross_valid
preds_test_all = []
for n in range(n_cross_valid):
x_train, x_valid, t_train, t_valid, x_test, x_test_id, _ = PreProcess()()
train_data = xgb.DMatrix(x_train, label=t_train)
 eval_data = xgb.DMatrix(x_valid, label=t_valid)
 test_data = xgb.DMatrix(x_test)
 params = {
'objective': 'multi:softprob',
'eval_metric': 'mlogloss',
'num_class': 9,
"booster": "gbtree",
"lambda": 0.00047596412767141483,
"alpha": 0.14634359955484605,
"max_depth": 8,
"eta": 0.04169411268469549,
"gamma": 0.00048700929022167433,
"grow_policy": "depthwise",
'tree_method': 'gpu_hist'
}
evals = [(eval_data, 'eval')]
bst = xgb.train(
 params,
train_data,
num_boost_round = 2000,
evals = evals,
early_stopping_rounds = 5 
)
 print('Best Score:{0:.4f}, Iteratin:{1:d}, Ntree_Limit:{2:d}'.format(
 bst.best_score, bst.best_iteration, bst.best_ntree_limit))
preds_test = bst.predict(test_data, ntree_limit=bst.best_ntree_limit)
 preds_test_all.append(preds_test)
 preds_test = np.mean(preds_test_all, axis=0)
 return preds_test<compute_train_metric>
train_sub = train[['id','place','date','ConfirmedCases','Fatalities']]
train_sub['logConfirmedCases'] = np.log(train_sub['ConfirmedCases'])
 train_sub = train_sub.set_index('date' )
def xgb_objective(trial):
x_train, x_valid, t_train, t_valid, _, _, _ = PreProcess()()
train_data = xgb.DMatrix(x_train, label=t_train)
 eval_data = xgb.DMatrix(x_valid, label=t_valid)
 params = {
'objective': 'multi:softprob',
'eval_metric': 'mlogloss',
'num_class': 9,
"booster": trial.suggest_categorical("booster", ["gbtree", "gblinear", "dart"]),
"lambda": trial.suggest_float("lambda", 1e-8, 1.0, log=True),
"alpha": trial.suggest_float("alpha", 1e-8, 1.0, log=True),
'tree_method': 'gpu_hist'
}
if params["booster"] == "gbtree" or params["booster"] == "dart":
params["max_depth"] = trial.suggest_int("max_depth", 1, 9)
 params["eta"] = trial.suggest_float("eta", 1e-8, 1.0, log=True)
 params["gamma"] = trial.suggest_float("gamma", 1e-8, 1.0, log=True)
 params["grow_policy"] = trial.suggest_categorical("grow_policy", ["depthwise", "lossguide"])
 if params["booster"] == "dart":
params["sample_type"] = trial.suggest_categorical("sample_type", ["uniform", "weighted"])
 params["normalize_type"] = trial.suggest_categorical("normalize_type", ["tree", "forest"])
 params["rate_drop"] = trial.suggest_float("rate_drop", 1e-8, 1.0, log=True)
 params["skip_drop"] = trial.suggest_float("skip_drop", 1e-8, 1.0, log=True)
 pruning_callback = optuna.integration.XGBoostPruningCallback(trial, f"eval-{params['eval_metric']}")
 evals = [(eval_data, 'eval')]
bst = xgb.train(
 params,
train_data,
num_boost_round = 500,
evals = evals,
early_stopping_rounds = 5,
callbacks=[pruning_callback] 
)
 return bst.best_score
def tune_xgb() :
pruner = optuna.pruners.MedianPruner(n_startup_trials=20, n_warmup_steps=20)
 study = optuna.create_study(direction="minimize", pruner = pruner)
 study.optimize(xgb_objective, n_trials=500)
 pruned_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.PRUNED]
complete_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.COMPLETE]
print("Study statistics: ")
 print(" Number of finished trials: ", len(study.trials))
print(" Number of pruned trials: ", len(pruned_trials))
print(" Number of complete trials: ", len(complete_trials))
print("Best trial:")
 trial = study.best_trial
print(" Value: {}".format(trial.value))
print(" Params: ")
 for key, value in trial.params.items() :
print(" {}: {}".format(key, value)) 
<save_to_csv>
list= []
for place in train_sub.place.unique() :
a = train_sub[train_sub['place']==place]
a['z_cases'] =(a['logConfirmedCases']- a['logConfirmedCases'].rolling(window=3 ).mean())/a['logConfirmedCases'].rolling(window=3 ).std()
a['zp_cases']= a['z_cases']- a['z_cases'].shift(3)
 a['z_death'] =(a['Fatalities']-a['Fatalities'].rolling(window=3 ).mean())/a['Fatalities'].rolling(window=3 ).std()
a['zp_death']= a['z_death']- a['z_death'].shift(3)
 list.append(a)
 rolling_df = pd.concat(list )
def submit(preds_test, x_test_id, alg):
df_sample_submission = pd.read_csv('/kaggle/input/otto-group-product-classification-challenge/sampleSubmission.csv')
 df_submission = pd.DataFrame(columns = df_sample_submission.columns)
 df_submission.id = x_test_id
df_submission[df_submission.columns[1:]] = preds_test
df_submission.to_csv('submission_{}.csv'.format(alg), index=False)
 def run() :
_, _, _, _, _, x_test_id, _ = PreProcess()()
n_cross_valid = 40
preds_test_all = []
preds_test_mlp = run_mlp(n_cross_valid)
 submit(preds_test_mlp, x_test_id, 'mlp')
 preds_test_all.append(preds_test_mlp)
 preds_test_lgb = run_lgb(n_cross_valid)
 submit(preds_test_lgb, x_test_id, 'lgb')
 preds_test_all.append(preds_test_lgb)
 preds_test_xgb = run_xgb(n_cross_valid)
 submit(preds_test_xgb, x_test_id, 'xgb')
 preds_test_all.append(preds_test_xgb)
 preds_test = np.mean(preds_test_all, axis=0)
 submit(preds_test, x_test_id, 'mlp_lgb_xgb')
 run()<set_options>
stationary_data =[]
for place in train_sub.place.unique() :
a= rolling_df[(rolling_df['place']==place)&(rolling_df['logConfirmedCases'] > 0)]['logConfirmedCases'].dropna()
try:
dftest = adfuller(a, autolag='AIC')
 if(dftest[1] < 0.001):
stationary_data.append(place)
 else:
pass
except:
pass
print(len(stationary_data))
%matplotlib inline
pd.options.display.max_columns = 999
plt.rcParams['figure.figsize'] =(14, 7)
 font = {'family' : 'verdana',
'weight' : 'bold',
'size' : 14}
plt.rc('font', **font)
 warnings.simplefilter("ignore")
 gc.enable()<load_from_csv>
station_death_data =[]
for place in train_sub.place.unique() :
dftest = adfuller(rolling_df[rolling_df['place']==place]['Fatalities'], autolag='AIC')
 if(dftest[1] < 0.001):
station_death_data.append(place)
 else:
pass
print(len(station_death_data))
train = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_train.gz', dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, 'visitId': np.int64})
 test = pd.read_csv('.. /input/create-extracted-json-fields-dataset/extracted_fields_test.gz', dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, 'visitId': np.int64})
 train.shape, test.shape<load_from_csv>
confirmedc_lag = ['Anhui_China', 'Chongqing_China','Guangdong_China',
'Guizhou_China', 'Hainan_China', 'Hebei_China','Hubei_China',
'Ningxia_China','Shandong_China','Shanxi_China', 'Sichuan_China']
train_store_1 = pd.read_csv('.. /input/exported-google-analytics-data/Train_external_data.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 train_store_2 = pd.read_csv('.. /input/exported-google-analytics-data/Train_external_data_2.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 test_store_1 = pd.read_csv('.. /input/exported-google-analytics-data/Test_external_data.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'})
 test_store_2 = pd.read_csv('.. /input/exported-google-analytics-data/Test_external_data_2.csv', low_memory=False, skiprows=6, dtype={"Client Id":'str'} )<data_type_conversions>
fatalities_lag = ['Hubei_China']
for df in [train_store_1, train_store_2, test_store_1, test_store_2]:
df["visitId"] = df["Client Id"].apply(lambda x: x.split('.', 1)[1] ).astype(np.int64 )<merge>
non_stationary_death_data = [ele for ele in allplaces]
for place in fatalities_lag:
if place in allplaces:
non_stationary_death_data.remove(place)
 print(len(non_stationary_death_data))
train = train.merge(pd.concat([train_store_1, train_store_2], sort=False), how="left", on="visitId")
 test = test.merge(pd.concat([test_store_1, test_store_2], sort=False), how="left", on="visitId")
 for df in [train, test]:
df.drop("Client Id", 1, inplace=True )<data_type_conversions>
train_sub['logConfirmedCases']= train_sub['logConfirmedCases'].replace(to_replace=-inf,
value=0 )
for df in [train, test]:
df["Revenue"].fillna('$', inplace=True)
 df["Revenue"] = df["Revenue"].apply(lambda x: x.replace('$', '' ).replace(',', ''))
df["Revenue"] = pd.to_numeric(df["Revenue"], errors="coerce")
 df["Revenue"].fillna(0.0, inplace=True )<drop_column>
poly_data = train[['date','place',
'ConfirmedCases','Fatalities']].merge(test[['date','place']],
how='outer',
on=['date','place'] ).sort_values(['place',
'date'])
 print(poly_data.date.min() , test.date.min() , train.date.max() , poly_data.date.max() )
for df in [train_store_1, train_store_2, test_store_1, test_store_2]:
del df
gc.collect()<groupby>
label = []
for place in poly_data.place.unique() :
labelrange = range(1,len(poly_data[poly_data['place']==place])+1)
 label.append([i for i in labelrange])
 lab = [item for lab in label for item in lab]
poly_data['label'] = lab
poly_data.head()
target_sums = train.groupby("fullVisitorId")["totals.transactionRevenue"].sum().reset_index()<count_unique_values>
XYtrain['intercept']= -1
result=pd.DataFrame()
for place in poly_data.place.unique() :
for degree in [2,3,4,5]:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['ConfirmedCases']
model = make_pipeline(PolynomialFeatures(degree), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(features))
rmsle = RMSLE(y_pred, target)
 result = result.append(pd.DataFrame({'place':[place],
'degree':[degree],'RMSLE': [rmsle]}))
ids_train = set(train.fullVisitorId.unique())
 ids_test = set(test.fullVisitorId.unique())
 print("Unique visitor ids in train:", len(ids_train))
print("Unique visitor ids in test:", len(ids_test))
print("Common visitors in train and test:", len(ids_train & ids_test))<count_values>
best_degree = pd.DataFrame()
for place in result.place.unique() :
a = result[result['place']==place]
best_degree = best_degree.append(a[a['RMSLE'] == a['RMSLE'].min() ])
 print(best_degree.groupby('degree')['place'].nunique())
 print('Zero polynomial(no fit): ',best_degree[best_degree['RMSLE']<0.00001]['place'].unique() )
problem = train[train.sessionId.map(train.sessionId.value_counts() == 2)].sort_values(["sessionId", 'visitStartTime'])
 problem.head(10 )<count_values>
fit_best_degree = best_degree[best_degree['RMSLE']>0.00001]
twodeg_places = fit_best_degree[fit_best_degree['degree']==2]['place'].unique()
threedeg_places = fit_best_degree[fit_best_degree['degree']==3]['place'].unique()
fourdeg_places = fit_best_degree[fit_best_degree['degree']==4]['place'].unique()
fivedeg_places = fit_best_degree[fit_best_degree['degree']==5]['place'].unique()
nofit_places1 = best_degree[best_degree['RMSLE']<0.00001]['place'].unique()
print(fit_best_degree.nunique())
 print(len(twodeg_places), len(threedeg_places),
len(fourdeg_places), len(fivedeg_places), len(nofit_places1))
( train.visitStartTime == train.visitId ).value_counts()<data_type_conversions>
XYtest = XYtest.reset_index(drop=True)
 XYtest['intercept'] = -1
train.loc[pd.to_datetime(train.visitStartTime, unit='s')== "2017-04-25 18:49:35"].head(8 )<count_values>
poly_predicted_confirmedcases = pd.DataFrame()
for place in twodeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['ConfirmedCases']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(2), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
a = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred),columns=['place','ConfirmedCases'])
 poly_predicted_confirmedcases = poly_predicted_confirmedcases.append(a)
 for place in threedeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['ConfirmedCases']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(3), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
b = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred.tolist()),columns=['place','ConfirmedCases'])
 poly_predicted_confirmedcases = poly_predicted_confirmedcases.append(b)
 for place in fourdeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['ConfirmedCases']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(4), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
c = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred.tolist()),columns=['place','ConfirmedCases'])
 poly_predicted_confirmedcases = poly_predicted_confirmedcases.append(c)
 for place in fivedeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['ConfirmedCases']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(5), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
d = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred.tolist()),columns=['place','ConfirmedCases'])
 poly_predicted_confirmedcases = poly_predicted_confirmedcases.append(d)
print("Train: ", np.bincount(train.visitId.value_counts()))<count_values>
fatalities_result=pd.DataFrame()
for place in poly_data.place.unique() :
for degree in [2,3,4,5]:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['Fatalities']
model = make_pipeline(PolynomialFeatures(degree), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(features))
rmsle = RMSLE(y_pred, target)
 fatalities_result = fatalities_result.append(pd.DataFrame({'place':[place],
'degree':[degree],'RMSLE': [rmsle]}))
print("test: ", np.bincount(test.visitId.value_counts()))<data_type_conversions>
fat_best_degree = pd.DataFrame()
for place in fatalities_result.place.unique() :
a = fatalities_result[fatalities_result['place']==place]
fat_best_degree = fat_best_degree.append(a[a['RMSLE'] == a['RMSLE'].min() ])
 print(fat_best_degree.groupby('degree')['place'].nunique())
 print('Zero polynomial(no fit): ',
fat_best_degree[fat_best_degree['RMSLE']<0.000001]['place'].unique() )
train.visitStartTime = pd.to_datetime(train.visitStartTime, unit='s')
 test.visitStartTime = pd.to_datetime(test.visitStartTime, unit='s')
 train["date"] = train.visitStartTime
test["date"] = test.visitStartTime<sort_values>
fit_best_degree = fat_best_degree[fat_best_degree['RMSLE']>0.000001]
twodeg_places = fit_best_degree[fit_best_degree['degree']==2]['place'].unique()
threedeg_places = fit_best_degree[fit_best_degree['degree']==3]['place'].unique()
fourdeg_places = fit_best_degree[fit_best_degree['degree']==4]['place'].unique()
fivedeg_places = fit_best_degree[fit_best_degree['degree']==5]['place'].unique()
nofit_places2 = fat_best_degree[fat_best_degree['RMSLE']<0.000001]['place'].unique()
print(fit_best_degree.nunique())
 print(len(twodeg_places), len(threedeg_places),
len(fourdeg_places), len(fivedeg_places), len(nofit_places2))
train.set_index("visitStartTime", inplace=True)
 test.set_index("visitStartTime", inplace=True)
 train.sort_index(inplace=True)
 test.sort_index(inplace=True )<count_values>
poly_predicted_fatalities = pd.DataFrame()
for place in twodeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['Fatalities']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(2), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
a = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred.tolist()),columns=['place','Fatalities'])
 poly_predicted_fatalities = poly_predicted_fatalities.append(a)
 for place in threedeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['Fatalities']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(3), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
b = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred.tolist()),columns=['place','Fatalities'])
 poly_predicted_fatalities = poly_predicted_fatalities.append(b)
 for place in fourdeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['Fatalities']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(4), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
c = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred.tolist()),columns=['place','Fatalities'])
 poly_predicted_fatalities = poly_predicted_fatalities.append(c)
 for place in fivedeg_places:
features = XYtrain[XYtrain['place']==place][['label','intercept']]
target = XYtrain[XYtrain['place']==place]['Fatalities']
Xtest = XYtest[XYtest['place']==place][['label','intercept']]
model = make_pipeline(PolynomialFeatures(5), Ridge())
 model.fit(np.array(features), target)
 y_pred = model.predict(np.array(Xtest))
d = pd.DataFrame(zip(XYtrain[XYtrain['place']==place]['place'],
y_pred.tolist()),columns=['place','Fatalities'])
 poly_predicted_fatalities = poly_predicted_fatalities.append(d)
def clearRare(columnname, limit = 1000):
vc = test[columnname].value_counts()
common = vc > limit
common = set(common.index[common].values)
 print("Set", sum(vc <= limit), columnname, "categories to 'other';", end=" ")
 train.loc[train[columnname].map(lambda x: x not in common), columnname] = 'other'
test.loc[test[columnname].map(lambda x: x not in common), columnname] = 'other'
print("now there are", train[columnname].nunique() , "categories in train" )<train_model>
for place in nofit_places1:
e = poly_data[(poly_data['place']==place)&(poly_data['date']>'2020-03-11')]
f = e['ConfirmedCases'].fillna(method = 'ffill')
 g = pd.DataFrame(zip(e['place'], f),columns=['place','ConfirmedCases'])
 poly_predicted_confirmedcases = poly_predicted_confirmedcases.append(g)
 for place in nofit_places2:
h = poly_data[(poly_data['place']==place)&(poly_data['date']>'2020-03-11')]
i = h['Fatalities'].fillna(method = 'ffill')
 j = pd.DataFrame(zip(h['place'], i),columns=['place','Fatalities'])
 poly_predicted_fatalities = poly_predicted_fatalities.append(j )
train.fillna(0, inplace=True)
 test.fillna(0, inplace=True )<drop_column>
poly_predicted_confirmedcases2= pd.DataFrame({'date':XYtest.date,
'place':poly_predicted_confirmedcases['place'].tolist() ,
'ConfirmedCases':poly_predicted_confirmedcases['ConfirmedCases'].tolist() })
 poly_predicted_confirmedcases2.head()
clearRare("device.browser")
 clearRare("device.operatingSystem")
 clearRare("geoNetwork.country")
 clearRare("geoNetwork.city")
 clearRare("geoNetwork.metro")
 clearRare("geoNetwork.networkDomain")
 clearRare("geoNetwork.region")
 clearRare("geoNetwork.subContinent")
 clearRare("trafficSource.adContent")
 clearRare("trafficSource.campaign")
 clearRare("trafficSource.keyword")
 clearRare("trafficSource.medium")
 clearRare("trafficSource.referralPath")
 clearRare("trafficSource.source" )<data_type_conversions>
poly_predicted_fatalities2= pd.DataFrame({'date':XYtest.date,
'place':poly_predicted_fatalities['place'].tolist() ,
'Fatalities':poly_predicted_fatalities['Fatalities'].tolist() })
 poly_predicted_fatalities2.head()
for df in [train, test]:
df["Avg.Session Duration"][df["Avg.Session Duration"] == 0] = "00:00:00"
df["Avg.Session Duration"] = df["Avg.Session Duration"].str.split(':' ).apply(lambda x: int(x[0])* 60 + int(x[1]))
df["Bounce Rate"] = df["Bounce Rate"].astype(str ).apply(lambda x: x.replace('%', '')).astype(float)
 df["Goal Conversion Rate"] = df["Goal Conversion Rate"].astype(str ).apply(lambda x: x.replace('%', '')).astype(float )<feature_engineering>
poly_compiled = poly_predicted_confirmedcases2.merge(poly_predicted_fatalities2, how='inner', on=['place','date'] )
for df in [train, test]:
df["id_incoherence"] = pd.to_datetime(df.visitId, unit='s')!= df.date
df["visitId_dublicates"] = df.visitId.map(df.visitId.value_counts())
 df["session_dublicates"] = df.sessionId.map(df.sessionId.value_counts() )<feature_engineering>
test_poly_compiled= test.merge(poly_compiled, how='inner', on=['place','date'])
 test_poly_compiled
for df in [train, test]:
df['weekday'] = df['date'].dt.dayofweek.astype(object)
 df['time'] = df['date'].dt.second + df['date'].dt.minute*60 + df['date'].dt.hour*3600
df['day'] = df['date'].dt.date<data_type_conversions>
submission= pd.read_csv(datapath+'submission.csv' )
df = pd.concat([train, test])
 df.sort_values(['fullVisitorId', 'date'], ascending=True, inplace=True)
 df['prev_session'] =(df['date'] - df[['fullVisitorId', 'date']].groupby('fullVisitorId')['date'].shift(1)).astype(np.int64)// 1e9 // 60 // 60
df['next_session'] =(df['date'] - df[['fullVisitorId', 'date']].groupby('fullVisitorId')['date'].shift(-1)).astype(np.int64)// 1e9 // 60 // 60
df.sort_index(inplace=True)
 train = df[:len(train)]
test = df[len(train):]<feature_engineering>
sub2 = submission[['ForecastId']].merge(test_poly_compiled[['ForecastId','ConfirmedCases','Fatalities']],
how='left',on='ForecastId' )
for df in [train, test]:
df['source.country'] = df['trafficSource.source'] + '_' + df['geoNetwork.country']
df['campaign.medium'] = df['trafficSource.campaign'] + '_' + df['trafficSource.medium']
df['browser.category'] = df['device.browser'] + '_' + df['device.deviceCategory']
df['browser.os'] = df['device.browser'] + '_' + df['device.operatingSystem']<feature_engineering>
sub2['ConfirmedCases'] = sub2['ConfirmedCases'].round(0)
 sub2['Fatalities'] = sub2['Fatalities'].round(0 ).abs()
for df in [train, test]:
df['device_deviceCategory_channelGrouping'] = df['device.deviceCategory'] + "_" + df['channelGrouping']
df['channelGrouping_browser'] = df['device.browser'] + "_" + df['channelGrouping']
df['channelGrouping_OS'] = df['device.operatingSystem'] + "_" + df['channelGrouping']
for i in ['geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country','geoNetwork.metro', 'geoNetwork.networkDomain', 'geoNetwork.region','geoNetwork.subContinent']:
for j in ['device.browser','device.deviceCategory', 'device.operatingSystem', 'trafficSource.source']:
df[i + "_" + j] = df[i] + "_" + df[j]
df['content.source'] = df['trafficSource.adContent'].astype(str)+ "_" + df['source.country']
df['medium.source'] = df['trafficSource.medium'] + "_" + df['source.country']<feature_engineering>
sub2.to_csv('submission.csv', index=False )
for feature in ["totals.hits", "totals.pageviews"]:
info = pd.concat([train, test], sort=False ).groupby("fullVisitorId")[feature].mean()
train["usermean_" + feature] = train.fullVisitorId.map(info)
 test["usermean_" + feature] = test.fullVisitorId.map(info)
 for feature in ["visitNumber"]:
info = pd.concat([train, test], sort=False ).groupby("fullVisitorId")[feature].max()
train["usermax_" + feature] = train.fullVisitorId.map(info)
 test["usermax_" + feature] = test.fullVisitorId.map(info )<define_variables>
excluded = ['date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue', 'visitId', 'visitStartTime',
'month', 'day', 'help']
cat_cols = [f for f in train.columns if(train[f].dtype == 'object' and f not in excluded)]
real_cols = [f for f in train.columns if(not f in cat_cols and f not in excluded)]<count_unique_values>
train[cat_cols].nunique()<data_type_conversions>
for col in cat_cols:
lbl = LabelEncoder()
lbl.fit(list(train[col].values.astype('str')) + list(test[col].values.astype('str')))
 train[col] = lbl.transform(list(train[col].values.astype('str')))
 test[col] = lbl.transform(list(test[col].values.astype('str')) )<data_type_conversions>
for col in real_cols:
train[col] = train[col].astype(float)
 test[col] = test[col].astype(float )<drop_column>