Kaggle competition: COVID19 Global Forecasting (Week 2)
class Config(object):
def __init__(self):
self.num_class = targets_tr.shape[1]
self.verbose=False
self.device = "cuda" if torch.cuda.is_available() else "cpu"
self.SPLITS = 10
self.EPOCHS = 200
self.num_ensembling = 1
self.seed = 0
self.cat_emb_dim=[1] * cat_tr.shape[1]
self.cats_idx = list(range(cat_tr.shape[1]))
self.cat_dims = [len(np.unique(cat_tr[:, i])) for i in range(cat_tr.shape[1])]
self.num_numericals= numerical_tr.shape[1]
self.save_name = ".. /output/kaggle/working/model"
self.strategy = "KFOLD"
cfg = Config() 
<concatenate>
sample_submission = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/submission.csv")
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv")
 train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv")
 pop = pd.read_csv("/kaggle/input/pop2020/data.csv")
 test2=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv")
 train2=pd.read_csv("/kaggle/input/pop2020/data.csv" )
X_test = np.concatenate([cat_test, numerical_test ], axis=1 )<choose_model_class>
pop.rename(columns={'Country or Area':'Country'}, inplace=True )
if cfg.strategy == "KFOLD":
oof_preds_all = []
oof_targets_all = []
scores_all = []
scores_auc_all= []
preds_test = []
for seed in range(cfg.num_ensembling):
print("
mskf = MultilabelStratifiedKFold(n_splits=cfg.SPLITS, random_state=cfg.seed+seed, shuffle=True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
p = []
for j,(train_idx, val_idx)in enumerate(mskf.split(np.zeros(len(cat_tr)) , targets_tr)) :
print("FOLDS : ", j)
 X_train, y_train = torch.as_tensor(np.concatenate([cat_tr[train_idx], numerical_tr[train_idx] ], axis=1)) , torch.as_tensor(targets_tr[train_idx])
 X_val, y_val = torch.as_tensor(np.concatenate([cat_tr[val_idx], numerical_tr[val_idx] ], axis=1)) , torch.as_tensor(targets_tr[val_idx])
 model = TabNetRegressor(n_d=24, n_a=24, n_steps=1, lambda_sparse=0,
cat_dims=cfg.cat_dims, cat_emb_dim=cfg.cat_emb_dim, cat_idxs=cfg.cats_idx,
optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2, weight_decay=1e-5),
mask_type='entmax', device_name=cfg.device,
scheduler_params=dict(milestones=[100, 150], gamma=0.9),
scheduler_fn=torch.optim.lr_scheduler.MultiStepLR)
 model.fit(X_train=X_train, y_train=y_train, X_valid=X_val, y_valid=y_val,max_epochs=cfg.EPOCHS, patience=50,
batch_size=1024, virtual_batch_size=128,
num_workers=0, drop_last=False, loss_fn=LabelSmoothing(0.001)
)
 preds = model.predict(X_val)
 preds = torch.sigmoid(torch.as_tensor(preds)).detach().cpu().numpy()
score = log_loss_multi(y_val, preds)
 temp = model.predict(X_test)
 p.append(torch.sigmoid(torch.as_tensor(temp)).detach().cpu().numpy())
 oof_preds.append(preds)
 oof_targets.append(y_val)
 scores.append(score)
 scores_auc.append(auc_multi(y_val,preds))
print(f"validation fold {j} : {score}")
 p = np.stack(p)
 preds_test.append(p)
 oof_preds_all.append(np.concatenate(oof_preds))
oof_targets_all.append(np.concatenate(oof_targets))
scores_all.append(np.array(scores))
scores_auc_all.append(np.array(scores_auc))
preds_test = np.stack(preds_test )<compute_test_metric>
map_state = {'US':'United States',
'Korea, South':'South Korea',
'Cabo Verde' : 'Carpe Verde',
'Congo(Brazzaville)': 'Republic of the Congo',
'Cote d'Ivoire':'Ivory Coast',
'Czechia':'Czech Republic',
'Eswatini':'Swaziland',
'Holy See':'Vatican City',
'North Macedonia':'Macedonia',
'Taiwan*':'Taiwan',
'occupied Palestinian territory':'Palestine'
}
map_state_rev = {v: k for k, v in map_state.items() }
if cfg.strategy == "KFOLD":
for i in range(cfg.num_ensembling):
print("CV score fold : ", log_loss_multi(oof_targets_all[i], oof_preds_all[i]))
print("auc mean : ", sum(scores_auc_all[i])/len(scores_auc_all[i]))<split>
pop['country'] = pop['country'].apply(lambda x: map_state_rev[x] if x in map_state_rev else x)
if cfg.strategy != "KFOLD":
i = 0
mskf = MultilabelStratifiedShuffleSplit(n_splits=1000, test_size=0.1, random_state=0)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
for j,(train_idx, val_idx)in enumerate(mskf.split(np.zeros(len(cat_tr)) , targets_tr)) :
if i == cfg.SPLITS:
break
if not check_targets(targets_tr[train_idx]):
continue
print("FOLDS : ", i, j)
 X_train, y_train = torch.as_tensor(np.concatenate([cat_tr[train_idx], numerical_tr[train_idx] ], axis=1)) , torch.as_tensor(targets_tr[train_idx])
 X_val, y_val = torch.as_tensor(np.concatenate([cat_tr[val_idx], numerical_tr[val_idx] ], axis=1)) , torch.as_tensor(targets_tr[val_idx])
 model = TabNetRegressor(n_d=8, n_a=8, n_steps=3, gamma=1.3, cat_dims=cat_dims, cat_emb_dim=cfg.cat_emb_dim, cat_idxs=cats_idx, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=1e-3, amsgrad=True), mask_type="sparsemax", device_name=cfg.device)
 name = cfg.save_name + f"_{j}"
model.load_model(name)
 preds = model.predict(X_val)
 preds = torch.sigmoid(torch.as_tensor(preds)).detach().cpu().numpy()
score = log_loss_multi(y_val, preds)
 temp = model.predict(X_test)
 p.append(torch.sigmoid(torch.as_tensor(temp)).detach().cpu().numpy())
 oof_preds.append(preds)
 oof_targets.append(y_val)
 scores.append(score)
 scores_auc.append(auc_multi(y_val,preds))
print(f"validation fold {j} : {score}")
 i+=1
p = np.stack(p)
 preds_test.append(p)
 preds_test = np.stack(preds_test)
<compute_test_metric>
train = pd.merge(train, pop, how='left', left_on = 'Country_Region', right_on = 'country' )
if cfg.strategy != "KFOLD":
print("auc mean : ", sum(scores_auc)/len(scores_auc))
print("CV score : ", log_loss_multi(np.concatenate(oof_targets), np.concatenate(oof_preds)) )<train_model>
train.isnull().sum()
print(model )<feature_engineering>
train.isnull().sum()
submission[columns] = preds_test.mean(1 ).mean(0)
 submission.loc[test_features['cp_type']=='ctl_vehicle', submission.columns[1:]] = 0<save_to_csv>
train.loc[:, train.isna().any() ]
submission.to_csv("submission.csv", index=False )<save_to_csv>
train.rename(columns={'Province_State':'Province'}, inplace=True)
 train.rename(columns={'Country_Region':'Country'}, inplace=True)
 train.rename(columns={'ConfirmedCases':'Confirmed'}, inplace=True )
submission.to_csv("submission.csv", index=False )<choose_model_class>
test.rename(columns={'Province_State':'Province'}, inplace=True)
 test.rename(columns={'Country_Region':'Country'}, inplace=True )
model.reducing_matrix<import_modules>
EMPTY_VAL = "EMPTY_VAL"
def fillState(state, country):
if Province == EMPTY_VAL: return country
return state
train['Province'].fillna(EMPTY_VAL, inplace=True)
 test['Province'].fillna(EMPTY_VAL, inplace=True )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import sklearn
import seaborn as sns
from sklearn.ensemble import RandomForestRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn.linear_model import LinearRegression, Perceptron, RidgeCV, Lasso
from sklearn.decomposition import PCA
from sklearn.metrics import log_loss
from sklearn.model_selection import train_test_split, KFold
from sklearn.feature_selection import VarianceThreshold
import torch
from torch import optim
from torch import nn
from torch.utils.data import Dataset, DataLoader
import torch.nn.functional as F 
<load_from_csv>
bridge_types =('Date', 'Province', 'Country', 'Confirmed',
'Id')
 countries = pd.DataFrame(train, columns=['Country'])
 state = pd.DataFrame(train, columns=['Province'])
 labelencoder = LabelEncoder()
train['Countries'] = labelencoder.fit_transform(train['Country'])
 train['State']= labelencoder.fit_transform(train['Province'])
 train
test['Countries'] = labelencoder.fit_transform(test['Country'])
 test['State']= labelencoder.fit_transform(test['Province'])
 train['Countries'].head()
train_X = pd.read_csv("/kaggle/input/lish-moa/train_features.csv")
 train_y = pd.read_csv("/kaggle/input/lish-moa/train_targets_scored.csv")
 train_y_ns = pd.read_csv("/kaggle/input/lish-moa/train_targets_nonscored.csv")
 test_X = pd.read_csv("/kaggle/input/lish-moa/test_features.csv")
 print(test_X.shape )<drop_column>
train['Date']= pd.to_datetime(train['Date'])
 test['Date']= pd.to_datetime(test['Date'])
train_ctl_vehicle_idx = train_X[train_X['cp_type']=='ctl_vehicle'].index
test_ctl_vehicle_idx = test_X[test_X['cp_type'] == 'ctl_vehicle'].index
train_X = train_X.drop(train_ctl_vehicle_idx ).reset_index(drop=True)
 train_y = train_y.drop(train_ctl_vehicle_idx ).reset_index(drop=True)
 test_X = test_X.drop(test_ctl_vehicle_idx ).reset_index(drop=True)
 train_X, test_X = train_X.drop('cp_type', axis=1), test_X.drop('cp_type', axis=1 )<define_variables>
train = train.set_index(['Date'])
 test = test.set_index(['Date'])
 train
g_features = [col for col in train_X.columns if col.startswith('g-')]
c_features = [col for col in train_X.columns if col.startswith('c-')]<drop_column>
def create_time_features(df):

 df['date'] = df.index
df['hour'] = df['date'].dt.hour
df['dayofweek'] = df['date'].dt.dayofweek
df['quarter'] = df['date'].dt.quarter
df['month'] = df['date'].dt.month
df['year'] = df['date'].dt.year
df['dayofyear'] = df['date'].dt.dayofyear
df['dayofmonth'] = df['date'].dt.day
df['weekofyear'] = df['date'].dt.weekofyear
X = df[['hour','dayofweek','quarter','month','year',
'dayofyear','dayofmonth','weekofyear']]
return X
def drop_cp(df):
return df[df['cp_type'] == 'trt_cp'].reset_index(drop=True)
<filter>
train.drop("date", axis=1, inplace=True)
 test.drop("date", axis=1, inplace=True )
test_ids = test_X.sig_id<concatenate>
train.isnull().sum()
def normal_split(train_X, train_y):
X_n_columns = len(train_X.columns)
 y_n_columns = len(train_y.columns)
 train_X_y = pd.merge(train_X, train_y ,on='sig_id')
 train_X = train_X_y.iloc[:, :X_n_columns]
train_y = train_X_y.iloc[:, X_n_columns:]
return train_X, train_y
train_X, train_y = normal_split(train_X, train_y)
 train_y.shape 
<categorify>
train.isnull().sum()
def preprocess(dataFrames, cat_features, con_features):
new_dfs = []
dataFrames_cat = [ df[cat_features] for df in dataFrames]
dataFrames_con = [ df[con_features] for df in dataFrames]
dataFrames_con = filter_variance(dataFrames_con)
 print("Variance filtered")
 print("High correlated features dropped")
 dataFrames_con = add_PCA(dataFrames_con, 30, 10)
 print("PCA added")
 dataFrames_con = add_new(dataFrames_con)
 print("Additional featires added")
 dataFrames_cat = map_categorical(dataFrames_cat)
 print("Categorical features transformed")
 for df_cat, df_con in zip(dataFrames_cat, dataFrames_con):
new_dfs.append(pd.concat(( df_cat, df_con), axis=1))
return new_dfs 
<categorify>
train.drop(['Country'], axis=1, inplace=True)
 train.drop(['Province'], axis=1, inplace=True )
<data_type_conversions>
test.drop(['Country'], axis=1, inplace=True)
 test.drop(['Province'], axis=1, inplace=True )
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 class MoADataset(Dataset):
def __init__(self, X, y=None):
if y is not None:
assert X.shape[0] == y.shape[0]
self.X = torch.tensor(X.to_numpy() , dtype=torch.float)
 self.y = None
if y is not None:
self.y = torch.tensor(y.to_numpy() , dtype=torch.float)
 def __getitem__(self, idx):
tensor_X = self.X[idx]
if self.y is not None:
tensor_y = self.y[idx]
return tensor_X, tensor_y
else:
return tensor_X
def __len__(self):
return len(self.X )<create_dataframe>
regressor = DecisionTreeRegressor(random_state = 0 )
dataset = MoADataset(train_X, train_y)
 train_dataloader = DataLoader(dataset, batch_size=10 )<choose_model_class>
from sklearn.metrics import mean_squared_error, mean_absolute_error
class MoANet(nn.Module):
def __init__(self, input_size, output_size):
super().__init__()
self.input_size = input_size
self.output_size = output_size
self.hidden_size = 256
self.wn = nn.utils.weight_norm
self.mlp = nn.Sequential(
 self.wn(nn.Linear(input_size, self.hidden_size)) ,
nn.Dropout(0.3),
nn.ELU() ,
self.wn(nn.Linear(self.hidden_size, self.hidden_size)) ,
nn.Dropout(0.3),
nn.ELU() ,
self.wn(nn.Linear(self.hidden_size, self.output_size)))
 def forward(self, x):
x = self.mlp(x)
 return x
net = MoANet(train_X.shape[1], train_y.shape[1] )<normalization>
x = train[['Countries','State','dayofweek','month','dayofyear','weekofyear']]
y1 = train[['Confirmed']]
y2 = train[['Fatalities']]
x_test = test[['Countries','State','dayofweek','month','dayofyear','weekofyear']]
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = F.leaky_relu(self.dense1(x), 1e-3)
 x = self.batch_norm2(x)
 x = self.dropout2(x)
 x =F.leaky_relu(self.dense2(x), 1e-3)
 x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x<train_model>
regressor.fit(x,y1)
 predict_1 = regressor.predict(x_test)
 predict_1 = pd.DataFrame(predict_1)
 predict_1.columns = ["Confirmed_predict"]
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for X, y in dataloader:
optimizer.zero_grad()
X, y = X.to(device), y.to(device)
 outputs = model(X)
 loss = loss_fn(outputs, y)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for X, y in dataloader:
X, y = X.to(device), y.to(device)
 outputs = model(X)
 loss = loss_fn(outputs, y)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for X in dataloader:
X = X.to(device)
 with torch.no_grad() :
outputs = model(X)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds 
<define_variables>
regressor.fit(x,y2)
 predict_2 = regressor.predict(x_test)
 predict_2 = pd.DataFrame(predict_2)
 predict_2.columns = ["Death_prediction"]
predict_2.head()
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 26
BATCH_SIZE = 256
LEARNING_RATE = 6e-4
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
HIDDEN_SIZE=2048<prepare_x_and_y>
Samle_submission = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/submission.csv")
 Samle_submission.columns
submission = Samle_submission[["ForecastId"]]
def run_training(fold_number, train_X, train_y, test_X, train_indexes, val_indexes, seed):
x_train = train_X.iloc[train_indexes].reset_index(drop=True)
 y_train = train_y.iloc[train_indexes].reset_index(drop=True)
 x_val = train_X.iloc[val_indexes].reset_index(drop=True)
 y_val = train_y.iloc[val_indexes].reset_index(drop=True)
 train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_val, y_val)
 trainloader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=train_X.shape[-1],
num_targets=train_y.shape[-1],
hidden_size=HIDDEN_SIZE, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e1,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( train_X.shape[0], train_y.shape[-1]))
best_loss = np.inf
print(f"FOLD: {fold_number}")
 for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"EPOCH: {epoch}, train_loss: {train_loss}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
early_step =0
best_loss = valid_loss
oof[val_indexes] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold_number}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
testdataset = MoADataset(test_X)
 testloader = DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=train_X.shape[-1],
num_targets=train_y.shape[-1],
hidden_size=HIDDEN_SIZE, 
)
 model.load_state_dict(torch.load(f"FOLD{fold_number}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_X), train_y.shape[-1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions<train_model>
Final_submission = pd.concat([predict_1,predict_2,submission],axis=1)
 Final_submission.head()
def run_k_fold(NFOLDS, train_X, train_y, test_X, seed):
oof = np.zeros(( train_y.shape))
predictions = np.zeros(( test_X.shape[0], train_y.shape[-1]))
kf = KFold(n_splits=NFOLDS, shuffle=True, random_state=seed)
 for fold,(train_indexes, val_indexes)in enumerate(kf.split(train_X)) :
oof_, pred_ = run_training(fold, train_X, train_y, test_X, train_indexes, val_indexes, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions<categorify>
Final_submission.columns = ['ConfirmedCases', 'Fatalities', 'ForecastId']
Final_submission = Final_submission[['ForecastId','ConfirmedCases', 'Fatalities']]
Final_submission["ConfirmedCases"] = Final_submission["ConfirmedCases"].astype(int)
 Final_submission["Fatalities"] = Final_submission["Fatalities"].astype(int )
SEED = [1973, 1998]
oof = np.zeros(( train_X.shape[0], train_y.shape[-1]))
predictions = np.zeros(( test_X.shape[0], train_y.shape[-1]))
for seed in SEED:
print(f"SEED: {seed}")
 oof_, predictions_ = run_k_fold(NFOLDS, train_X, train_y, test_X, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
<create_dataframe>
Final_submission.to_csv("submission.csv",index=False)
 print('Model ready for submission!' )
train_prediction_df= pd.DataFrame(data=oof, columns=train_y.columns)
 test_prediction_df = pd.DataFrame(data=predictions, columns=train_y.columns )<load_from_csv>
m1 = RandomForestRegressor(n_estimators=100,max_depth=20)
 m1.fit(x,y1 )
train_ctl_vehicle_idx
test_ctl_vehicle_idx
tts = pd.read_csv("/kaggle/input/lish-moa/train_targets_scored.csv")
 train = pd.read_csv("/kaggle/input/lish-moa/train_features.csv")
 tstf = pd.read_csv("/kaggle/input/lish-moa/test_features.csv")
 print(tts.shape)
 dropped_sig_id_train = pd.DataFrame(tts.iloc[:,0].drop(train_ctl_vehicle_idx, axis=0 ).reset_index(drop=True), columns=['sig_id'])
 train_prediction_df = pd.concat([dropped_sig_id_train, train_prediction_df], axis=1)
 valid_results = tts.drop(columns=train_y.columns ).merge(train_prediction_df, on='sig_id', how='left' ).fillna(0)
 y_true = tts[train_y.columns].values
y_pred = valid_results[train_y.columns].values
score = 0
for i in range(y_true.shape[1]):
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_
score /= train_y.shape[1]
print("CV log_loss: ", score)
<merge>
pred1=m1.predict(x_test )
dropped_sig_id_test = pd.DataFrame(tstf.iloc[:,0].drop(test_ctl_vehicle_idx, axis=0 ).reset_index(drop=True), columns=['sig_id'])
 test_prediction_df = pd.concat([dropped_sig_id_test, test_prediction_df], axis=1)
 test_results = tstf.drop(columns=tstf.columns[1:] ).merge(test_prediction_df, on='sig_id', how='left' ).fillna(0)
 test_results.shape<save_to_csv>
m1.fit(x,y2 )
test_results.to_csv("submission.csv", index=None )<train_model>
pred2=m1.predict(x_test)
 pred2
<save_to_csv>
pred1 = pd.DataFrame(pred1)
 pred1.columns = ["ConfirmedCases"]
<predict_on_test>
pred2 = pd.DataFrame(pred2)
 pred2.columns = ["Fatalities"]
def one_to_all_reg(reg_model, train_X, train_y):
models = {}
for idx, target_name in enumerate(train_y.columns):
print("Train
targets = train_y[target_name]
reg_model.fit(train_X, targets)
 models[target_name] = reg_model
return models
def predict_ensemble(test_X, models, targets_names):
df_prediction = pd.DataFrame()
for target_name in targets_names:
df_prediction[target_name] = models[target_name].predict(test_X)
 return df_prediction
def create_equal_ordering(pred_df, true_df):
assert pred_df.shape == true_df.shape
numpy_pred = np.zeros(pred_df.shape)
 numpy_true = np.zeros(true_df.shape)
 for idx, target_name in enumerate(true_df.columns):
numpy_pred[:, idx] = pred_df[target_name].to_numpy()
numpy_true[:, idx] = true_df[target_name].to_numpy()
return numpy_pred, numpy_true<init_hyperparams>
Sample_submission = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/submission.csv")
 Sample_submission.columns
submission = Sample_submission[["ForecastId"]]
<import_modules>
RF_submission = pd.concat([pred1,pred2,submission],axis=1)
 RF_submission.head()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import torch
import torch.nn as nn
from torch.utils.data import DataLoader, TensorDataset
from sklearn.model_selection import train_test_split<choose_model_class>
RF_submission.to_csv("submission.csv",index=False)
 print('Model ready for submission!' )
class MyModel(nn.Module):
def __init__(self, in_features, out_features):
super().__init__()
self.in_features = in_features
self.out_features = out_features
self.model = nn.Sequential(
 nn.BatchNorm1d(in_features),
nn.Dropout(0.5),
nn.Linear(in_features, 2048),
nn.ReLU() ,
nn.BatchNorm1d(2048),
nn.Dropout(0.5),
nn.Linear(2048, 1024),
nn.ReLU() ,
nn.BatchNorm1d(1024),
nn.Dropout(0.5),
nn.Linear(1024, out_features),
nn.Sigmoid() 
)
 def forward(self, x):
return self.model(x )<choose_model_class>
import keras
def train_model(model, train_dataloader, test_dataloader=None, optimizer=None, scheduler=None, max_epoch=10, criterion=None):
if optimizer is None:
optimizer = torch.optim.Adam(model.parameters())
 if scheduler is None:
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 0.1, 2)
 if criterion is None:
criterion = nn.BCELoss()
for epoch in range(max_epoch):
model.train()
for i,(x_batch, y_batch)in enumerate(train_dataloader):
preds = model(x_batch)
 optimizer.zero_grad()
loss = criterion(preds, y_batch)
 loss.backward()
optimizer.step()
scheduler.step()
if i % 20 == 0:
print(f'Epoch: {epoch}, train loss: {loss.item() :12.5f}')
 model.eval()
with torch.no_grad() :
train, y_train = train_dataloader.dataset.tensors
train_preds = model(train)
 train_loss = criterion(train_preds, y_train ).item()
if test_dataloader is not None:
test, y_test = test_dataloader.dataset.tensors
test_preds = model(test)
 test_loss = criterion(test_preds, y_test ).item()
print(f'Epoch: {epoch} final', f'test loss: {test_loss}', f'train loss: {train_loss}')
 else:
print(f'Epoch: {epoch} final: train loss: {train_loss}' )<train_model>
model = Sequential()
model.add(Dense(100, activation='relu', input_dim=5))
model.add(Dense(1))
model.compile(optimizer='adam', loss='mse')
def train_ensemble(X, y, base_model=MyModel, n_models=5, max_epoch=50):
criterion = nn.BCELoss()
kf = KFold(n_splits=n_models, shuffle=True, random_state=48)
 models = []
for i,(train_ind, val_ind)in enumerate(kf.split(X, y)) :
print(f'Training model {i+1}/{kf.n_splits}')
 x_train = torch.tensor(X.iloc[train_ind,:].values, dtype=torch.float32)
 y_train = torch.tensor(y.iloc[train_ind,:].values, dtype=torch.float32)
 x_val = torch.tensor(X.iloc[val_ind,:].values, dtype=torch.float32)
 y_val = torch.tensor(y.iloc[val_ind,:].values, dtype=torch.float32)
 train_dataloader = DataLoader(TensorDataset(x_train, y_train), batch_size=128, shuffle=True)
 val_dataloader = DataLoader(TensorDataset(x_val, y_val), batch_size=128)
 best_loss = np.inf
best_model = None
cur_model = base_model(x_train.shape[1], y_train.shape[1] ).to('cuda')
 optimizer = torch.optim.Adam(cur_model.parameters() ,weight_decay=1e-5)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=3, eps=1e-4)
 for epoch in range(max_epoch):
cur_model.train()
for i,(x_batch, y_batch)in enumerate(train_dataloader):
preds = cur_model(x_batch.to('cuda'))
optimizer.zero_grad()
loss = criterion(preds, y_batch.to('cuda'))
loss.backward()
optimizer.step()
cur_model.eval()
with torch.no_grad() :
train_preds = cur_model(x_train.to('cuda'))
train_loss = criterion(train_preds, y_train.to('cuda')).item()
val_preds = cur_model(x_val.to('cuda'))
val_loss = criterion(val_preds, y_val.to('cuda')).item()
print(f'Epoch {epoch}: train loss: {train_loss}, val loss: {val_loss}')
 scheduler.step(val_loss)
 if val_loss < best_loss:
best_model = deepcopy(cur_model)
 best_loss = val_loss
models += [best_model]
return models<define_variables>
model.fit(x, y1, epochs=20, verbose=0)
 yhat = model.predict(x_test, verbose=0)
 print(yhat )
best_features = ['g-392', 'c-65', 'g-100', 'c-9', 'g-50', 'c-79', 'c-98', 'g-37',
'c-6', 'c-26', 'g-439', 'g-628', 'g-744', 'g-351', 'g-298', 'c-42',
'g-410', 'g-761', 'c-18', 'g-418', 'g-146', 'c-57', 'c-64',
'g-322', 'c-48', 'c-38', 'c-82', 'g-63', 'g-534', 'c-92', 'g-186',
'c-28', 'g-486', 'c-70', 'g-672', 'g-91', 'g-731', 'c-10', 'g-386',
'g-121', 'g-443', 'g-206', 'g-723', 'c-81', 'c-36', 'c-33', 'g-85',
'g-235', 'g-406', 'g-683', 'c-52', 'c-62', 'c-63', 'g-365', 'c-21',
'c-60', 'c-15', 'c-66', 'g-629', 'c-49', 'g-248', 'c-59', 'c-24',
'c-76', 'g-669', 'g-106', 'g-38', 'g-140', 'c-30', 'c-22', 'g-72',
'c-25', 'c-23', 'c-8', 'c-83', 'g-489', 'g-369', 'c-47', 'g-158',
'g-297', 'g-147', 'c-5', 'c-77', 'g-163', 'g-332', 'g-344', 'c-50',
'g-335', 'c-2', 'g-503', 'g-208', 'g-152', 'c-17', 'c-41', 'g-353',
'c-34', 'g-664', 'c-96', 'g-228', 'c-67', 'g-569', 'g-750', 'g-30',
'g-578', 'c-90', 'c-72', 'g-257', 'c-75', 'c-97', 'g-98', 'g-500',
'c-1', 'g-728', 'c-44', 'g-360', 'c-85', 'g-195', 'c-31', 'c-11',
'c-40', 'g-135', 'g-65', 'c-95', 'c-80', 'g-261', 'g-590', 'c-54',
'c-51', 'c-13', 'c-12', 'g-201', 'g-83', 'g-468', 'g-58', 'g-478',
'g-460', 'g-574', 'c-45', 'c-94', 'c-4', 'g-367', 'c-69', 'g-407',
'c-73', 'g-349', 'g-155', 'g-113', 'g-350', 'c-91', 'g-546',
'g-131', 'g-52', 'g-745', 'c-55', 'c-27', 'c-14', 'g-379', 'g-51',
'g-199', 'g-241', 'g-568', 'g-10', 'c-93', 'g-508', 'c-84', 'c-78',
'g-433', 'c-20', 'c-39', 'g-7', 'g-177', 'g-185']
def preprocess(features, target=None, is_train=True):
out_features = features.drop(['sig_id'], axis=1 ).copy()
out_features['cp_type'] =(features.cp_dose == 'trt_cp' ).astype(float)
 out_features['cp_dose'] =(features.cp_dose == 'D1' ).astype(float)
 out_features['cp_time'] =(features.cp_time == 48 ).astype(float)
 out_target = None
if target is not None:
out_target = target.drop('sig_id', axis=1 ).copy()
return out_features, out_target<load_from_csv>
model.fit(x, y2, epochs=20, verbose=0)
 yhat1 = model.predict(x_test, verbose=0)
 print(yhat1 )
data_dir = '.. /input/lish-moa/'
train_features = pd.read_csv(data_dir+'train_features.csv')
 test_features = pd.read_csv(data_dir+'test_features.csv')
 train_target = pd.read_csv(data_dir+'train_targets_scored.csv')
<prepare_x_and_y>
Sample_submission = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/submission.csv")
 Sample_submission.columns
submission_mlp = Sample_submission[["ForecastId"]]
x_train, y_train = preprocess(train_features, train_target)
 test_zero_mask = test_features.cp_type == 'ctl_vehicle'
x_test, _ = preprocess(test_features, is_train=False )<categorify>
cases=pd.DataFrame(yhat)
 cases.columns=["ConfirmedCases"]
fatal=pd.DataFrame(yhat1)
 fatal.columns=["Fatalities"]
Final_submission = pd.concat([cases,fatal,submission_mlp],axis=1)
 Final_submission.head()
class Autoencoder(nn.Module):
def __init__(self, n_inputs, dropout=0.1):
super().__init__()
self.encoder = nn.Sequential(
 nn.BatchNorm1d(n_inputs),
nn.Dropout(dropout),
nn.Linear(n_inputs, n_inputs // 2),
nn.ReLU() ,
nn.BatchNorm1d(n_inputs // 2),
nn.Dropout(dropout),
nn.Linear(n_inputs // 2, n_inputs // 4),
nn.ReLU() ,
nn.BatchNorm1d(n_inputs // 4),
nn.Dropout(dropout),
nn.Linear(n_inputs // 4, n_inputs // 8),
nn.ReLU() 
)
 self.decoder = nn.Sequential(
 nn.Linear(n_inputs // 8, n_inputs // 4),
nn.ReLU() ,
nn.BatchNorm1d(n_inputs // 4),
nn.Dropout(dropout),
nn.Linear(n_inputs // 4, n_inputs // 2),
nn.ReLU() ,
nn.BatchNorm1d(n_inputs // 2),
nn.Dropout(dropout),
nn.Linear(n_inputs // 2, n_inputs)
)
 def forward(self, x):
encoded = self.encoder(x)
 decoded = self.decoder(encoded)
 return decoded<define_variables>
Final_submission.to_csv("submission.csv",index=False)
 print('Model ready for submission!' )
gen_f = [col for col in x_train.columns if col.startswith('g-')]
cell_f = [col for col in x_train.columns if col.startswith('c-')]<split>
import xgboost as xgb
from xgboost import plot_importance, plot_tree
from sklearn.metrics import mean_squared_error, mean_absolute_error
x_train_cell, x_test_cell = train_test_split(x_train[cell_f], test_size = 0.2, random_state=111 )<create_dataframe>
reg = xgb.XGBRegressor(n_estimators=1000)
 reg.fit(x, y1,
verbose=False )
train_dataset = TensorDataset(torch.tensor(x_train_cell.values, dtype=torch.float32),
torch.tensor(x_train_cell.values, dtype=torch.float32))
test_dataset = TensorDataset(torch.tensor(x_test_cell.values, dtype=torch.float32),
torch.tensor(x_test_cell.values, dtype=torch.float32))
train_dataloader = DataLoader(train_dataset, batch_size=256)
 test_dataloader = DataLoader(test_dataset, batch_size=256 )<categorify>
reg = xgb.XGBRegressor(n_estimators=1000)
 reg.fit(x, y2,
verbose=False )
cells_autoencoder = Autoencoder(len(cell_f))
optimizer = torch.optim.Adam(cells_autoencoder.parameters() , weight_decay=1e-3)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=3, eps=1e-4)
 train_model(cells_autoencoder, train_dataloader, test_dataloader, max_epoch=30, criterion=nn.MSELoss() )<categorify>
XG_fatalities = reg.predict(x_test )
with torch.no_grad() :
ae_cells_train = cells_autoencoder.encoder(torch.tensor(x_train[cell_f].values, dtype=torch.float32)).numpy()<split>
XG_Confirmed = reg.predict(x_test)
x_train_gen, x_test_gen = train_test_split(x_train[gen_f], test_size = 0.2, random_state=111 )<create_dataframe>
Sample_submission = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/submission.csv")
 Sample_submission.columns
submission_xg = Sample_submission[["ForecastId"]]
train_dataset = TensorDataset(torch.tensor(x_train_gen.values, dtype=torch.float32),
torch.tensor(x_train_gen.values, dtype=torch.float32))
test_dataset = TensorDataset(torch.tensor(x_test_gen.values, dtype=torch.float32),
torch.tensor(x_test_gen.values, dtype=torch.float32))
train_dataloader = DataLoader(train_dataset, batch_size=256)
 test_dataloader = DataLoader(test_dataset, batch_size=256 )<categorify>
cases=pd.DataFrame(XG_Confirmed)
 cases.columns=["ConfirmedCases"]
fatal=pd.DataFrame(XG_fatalities)
 fatal.columns=["Fatalities"]
Final_submission = pd.concat([cases,fatal,submission_xg],axis=1)
 Final_submission.head()
gen_autoencoder = Autoencoder(len(gen_f))
optimizer = torch.optim.Adam(gen_autoencoder.parameters() , weight_decay=1e-3)
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=3, eps=1e-4)
 train_model(gen_autoencoder, train_dataloader, test_dataloader, max_epoch=30, criterion=nn.MSELoss() )<categorify>
param_grid = {'n_estimators': [1000]}
with torch.no_grad() :
ae_gen_train = gen_autoencoder.encoder(torch.tensor(train_features[gen_f].values, dtype=torch.float32)).numpy()<create_dataframe>
def gridSearchCV(model, X_Train, y_Train, param_grid, cv=10, scoring='neg_mean_squared_error'):
start = time.time()
grid_cv = GridSearchCV(model, param_grid, cv=10, scoring="neg_mean_squared_error")
 grid_cv.fit(X_Train, y_Train)
 print(f'{type(model ).__name__} Hyper Paramter Tuning took a Time: {time.time() - start}')
 print(f'Best {scoring}: {grid_cv.best_score_}')
 print("Best Hyper Parameters:
{}".format(grid_cv.best_params_))
return grid_cv.best_estimator_
train_enc = pd.DataFrame(np.concatenate(( ae_gen_train, ae_cells_train),axis=1))
train_enc = pd.concat(( x_train, train_enc), axis=1 )<train_model>
model = XGBRegressor()
model1 = gridSearchCV(model, x, y1, param_grid, 10, 'neg_mean_squared_error')
 model2 = gridSearchCV(model, x, y2, param_grid, 10, 'neg_mean_squared_error' )
ensemble = train_ensemble(train_enc, y_train )<save_model>
y1_pred = model1.predict(x_test)
 y1_pred = y1_pred.round()
np.save('best_ensemble_pca.npy', ensemble )<load_pretrained>
y2_pred = model2.predict(x_test)
 y2_pred = y2_pred.round()
ensemble = np.load('best_ensemble_pca.npy', allow_pickle=True )<categorify>
Sample_submission = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/submission.csv")
 Sample_submission.columns
submission_xg = Sample_submission[["ForecastId"]]
<concatenate><EOS> 
cases=pd.DataFrame(y1_pred)
 cases.columns=["ConfirmedCases"]
fatal=pd.DataFrame(y2_pred)
 fatal.columns=["Fatalities"]
Final_submission = pd.concat([cases,fatal,submission_xg],axis=1)
 Final_submission.head(20 )