Kaggle competition: Mechanisms of Action (MoA) Prediction
sub1 = get_cpmp_sub()
sub1['ForecastId'] = sub1['ForecastId'].astype('int' )<sort_values>
warnings.filterwarnings('ignore' )
sub1.sort_values("ForecastId", inplace=True)
 sub2.sort_values("ForecastId", inplace=True )<compute_test_metric>
from sklearn.preprocessing import QuantileTransformer
TARGETS = ["ConfirmedCases", "Fatalities"]
[np.sqrt(mean_squared_error(np.log1p(sub1[t].values), np.log1p(sub2[t].values)))for t in TARGETS]<save_to_csv>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 df = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
sub_df = sub1.copy()
for t in TARGETS:
sub_df[t] = np.expm1(np.log1p(sub1[t].values)*0.5 + np.log1p(sub2[t].values)*0.5)
 sub_df.to_csv("submission.csv", index=False )<define_variables>
train_features2=train_features.copy()
test_features2=test_features.copy()
week = 4<import_modules>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
import numpy as np
import pandas as pd
import datetime
import os
import matplotlib.pyplot as plt<feature_engineering>
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
train = pd.read_csv(f"/kaggle/input/covid19-global-forecasting-week-{week}/train.csv")
 train.loc[:, "geo"] = np.where(train.loc[:, "Province_State"].isna() , train.loc[:, "Country_Region"], train.loc[:, "Country_Region"] + "_" + train.loc[:, "Province_State"])
 train.loc[:, 'Date'] = pd.to_datetime(train.loc[:, 'Date'])
 train = train.loc[train.loc[:, 'Date'] > '2020-02-20', :]
train_last_date = train.Date.unique() [-1]
print(f"Dataset has training data untill : {str(train_last_date)[:10]}")
 print(f"Training dates: {len(train.Date.unique())}" )<feature_engineering>
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )
all_countries = train.groupby(['Date'] ).agg({'ConfirmedCases': np.sum} ).reset_index()
all_countries['sh'] = train.groupby(['Date'] ).agg({'ConfirmedCases': np.sum} ).reset_index().shift(1)['ConfirmedCases']
all_countries['sh'] =(all_countries['ConfirmedCases'] / all_countries['sh'])- 1
all_countries = all_countries.set_index("Date")
 all_countries['sh'].plot()<filter>
n_comp = 600
pca_g = PCA(n_components=n_comp, random_state=42)
 data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 gpca=(pca_g.fit(data[GENES]))
train2=(gpca.transform(train_features[GENES]))
test2 =(gpca.transform(test_features[GENES]))
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1)
 dump(gpca, open('gpca.pkl', 'wb'))
train_old = train.loc[train.loc[:, 'Date'] > '2020-04-01', :]<load_from_csv>
n_comp = 50
pca_c = PCA(n_components=n_comp, random_state=42)
 data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 cpca=(pca_c.fit(data[CELLS]))
train2=(cpca.transform(train_features[CELLS]))
test2 =(cpca.transform(test_features[CELLS]))
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1)
 dump(cpca, open('cpca.pkl', 'wb'))
test = pd.read_csv(f"/kaggle/input/covid19-global-forecasting-week-{week}/test.csv")
 test.loc[:, 'Date'] = pd.to_datetime(test.loc[:, 'Date'])
 test_first_date = test.loc[:, 'Date'].values[0]
test_last_date = test.loc[:, 'Date'].values[-1]
print(f'Test period from {str(test_first_date)[:10]} to {str(test_last_date)[:10]}' )<data_type_conversions>
c_n = [f for f in list(train_features.columns)if f not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train_features[c_n].var() >= 0.85 ).values
tmp = train_features[c_n].loc[:, mask]
train_features = pd.concat([train_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 tmp = test_features[c_n].loc[:, mask]
test_features = pd.concat([test_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1 )
period =(np.array(test_last_date, dtype='datetime64[D]' ).astype(np.int64)- np.array(train_last_date, dtype='datetime64[D]' ).astype(np.int64))<data_type_conversions>
def fe_cluster_genes(train, test, n_clusters_g = 22, SEED = 42):
features_g = GENES
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans_genes = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 dump(kmeans_genes, open('kmeans_genes.pkl', 'wb'))
train[f'clusters_{kind}'] = kmeans_genes.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_genes.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 return train, test
train_features2 ,test_features2=fe_cluster_genes(train_features2,test_features2 )
print(f"Prediction days: {(np.array(test_last_date, dtype='datetime64[D]' ).astype(np.int64)- np.array(train_last_date, dtype='datetime64[D]' ).astype(np.int64)) +1}")
 print(f"Public set: {(np.array(train_last_date, dtype='datetime64[D]' ).astype(np.int64)- np.array(test_first_date, dtype='datetime64[D]' ).astype(np.int64)) +1}")
 print(f"Full prediction set: {(np.array(test_last_date, dtype='datetime64[D]' ).astype(np.int64)- np.array(test_first_date, dtype='datetime64[D]' ).astype(np.int64)) +1}" )<define_variables>
def fe_cluster_cells(train, test, n_clusters_c = 4, SEED = 42):
features_c = CELLS
def create_cluster(train, test, features, kind = 'c', n_clusters = n_clusters_c):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans_cells = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 dump(kmeans_cells, open('kmeans_cells.pkl', 'wb'))
train[f'clusters_{kind}'] = kmeans_cells.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_cells.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster_cells(train_features2,test_features2 )
win = 32
hor = 2<feature_engineering>
train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1 )
train.loc[:, 'ConfirmedCases'] = train.loc[:, 'ConfirmedCases'] - train.groupby('geo')['ConfirmedCases'].shift(periods=1)
 train.loc[:, 'Fatalities'] = train.loc[:, 'Fatalities'] - train.groupby('geo')['Fatalities'].shift(periods=1)
 train = train.groupby('geo' ).tail(train.groupby('geo' ).size().values[0]-1)
 train.loc[train.loc[:, 'ConfirmedCases'] < 0, 'ConfirmedCases'] = 0.0
train.loc[train.loc[:, 'Fatalities'] < 0, 'Fatalities'] = 0.0
train<count_values>
def fe_cluster_pca(train, test,n_clusters=5,SEED = 42):
data=pd.concat([train,test],axis=0)
 kmeans_pca = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 dump(kmeans_pca, open('kmeans_pca.pkl', 'wb'))
train[f'clusters_pca'] = kmeans_pca.predict(train.values)
 test[f'clusters_pca'] = kmeans_pca.predict(test.values)
 train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = fe_cluster_pca(train_pca,test_pca )
train.Country_Region.value_counts()<install_modules>
train_cluster_pca = train_cluster_pca.iloc[:,650:]
test_cluster_pca = test_cluster_pca.iloc[:,650:]
%%bash
pip install pytorch_lightning<import_modules>
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]
import torch.nn as nn
import torch
import torch.nn.functional as F
import pytorch_lightning as ptl
from torch import optim
from torch.utils.data import DataLoader
from collections import OrderedDict<compute_test_metric>
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
def rmsle(predict, target):
return torch.sqrt(((torch.log(predict + 1)- torch.log(target + 1)) **2 ).mean() )<create_dataframe>
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-26'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
df['c26_c38'] = df['c-26'] * df['c-38']
df['c90_c13'] = df['c-90'] * df['c-13']
df['c85_c31'] = df['c-85'] * df['c-31']
df['c63_c42'] = df['c-63'] * df['c-42']
df['c94_c11'] = df['c-94'] * df['c-11']
df['c94_c60'] = df['c-94'] * df['c-60']
df['c55_c42'] = df['c-55'] * df['c-42']
df['g37_c50'] = df['g-37'] * df['g-50']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2 )
class MTSFDataset(torch.utils.data.Dataset):
def __init__(self, window, horizon, set_type, tra, validation):
assert type(set_type)== type('str')
 self.window = window
self.horizon = horizon
self.tra = tra
self.validation = validation
self.set_type = set_type
if set_type == 'train':
rawdata = tra
elif set_type == 'validation':
rawdata = validation
_, self.len, self.var_num = rawdata.shape
self.sample_num = max(self.len - self.window - self.horizon + 1, 0)
 self.samples, self.labels = self.__getsamples(rawdata)
 def __getsamples(self, data):
x, y = [], []
for j in range(len(data)) :
X = torch.zeros(( self.sample_num, self.window, self.var_num))
Y = torch.zeros(( self.sample_num, 1, self.var_num))
for i in range(self.sample_num):
start = i
end = i + self.window
X[i, :, :] = torch.from_numpy(data[j, start:end, :])
 Y[i, :, :] = torch.from_numpy(data[j, end+self.horizon-1, :])
 x.append(X)
 y.append(Y)
 return(torch.cat(x), torch.cat(y))
def __len__(self):
return self.sample_num
def __getitem__(self, idx):
sample = [self.samples[idx, :, :], self.labels[idx, :, :]]
return sample<compute_train_metric>
train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:]
class ScaledDotProductAttention(nn.Module):
def __init__(self, temperature, attn_dropout=0.1):
super().__init__()
self.temperature = temperature
self.dropout = nn.Dropout(attn_dropout)
 self.softmax = nn.Softmax(dim=2)
 def forward(self, q, k, v):
attn = torch.bmm(q, k.transpose(1, 2))
attn = attn / self.temperature
attn = self.softmax(attn)
 attn = self.dropout(attn)
 output = torch.bmm(attn, v)
 return output, attn
class MultiHeadAttention(nn.Module):
def __init__(self, n_head, d_model, d_k, d_v, dropout=0.1):
super().__init__()
self.n_head = n_head
self.d_k = d_k
self.d_v = d_v
self.w_qs = nn.Linear(d_model, n_head * d_k)
 self.w_ks = nn.Linear(d_model, n_head * d_k)
 self.w_vs = nn.Linear(d_model, n_head * d_v)
 nn.init.normal_(self.w_qs.weight, mean=0, std=np.sqrt(2.0 /(d_model + d_k)))
 nn.init.normal_(self.w_ks.weight, mean=0, std=np.sqrt(2.0 /(d_model + d_k)))
 nn.init.normal_(self.w_vs.weight, mean=0, std=np.sqrt(2.0 /(d_model + d_v)))
 self.attention = ScaledDotProductAttention(temperature=np.power(d_k, 0.5))
self.layer_norm = nn.LayerNorm(d_model)
 self.fc = nn.Linear(n_head * d_v, d_model)
 nn.init.xavier_normal_(self.fc.weight)
 self.dropout = nn.Dropout(dropout)
 def forward(self, q, k, v):
d_k, d_v, n_head = self.d_k, self.d_v, self.n_head
sz_b, len_q, _ = q.size()
sz_b, len_k, _ = k.size()
sz_b, len_v, _ = v.size()
residual = q
q = self.w_qs(q ).view(sz_b, len_q, n_head, d_k)
 k = self.w_ks(k ).view(sz_b, len_k, n_head, d_k)
 v = self.w_vs(v ).view(sz_b, len_v, n_head, d_v)
 q = q.permute(2, 0, 1, 3 ).contiguous().view(-1, len_q, d_k)
 k = k.permute(2, 0, 1, 3 ).contiguous().view(-1, len_k, d_k)
 v = v.permute(2, 0, 1, 3 ).contiguous().view(-1, len_v, d_v)
 output, attn = self.attention(q, k, v)
 output = output.view(n_head, sz_b, len_q, d_v)
 output = output.permute(1, 2, 0, 3 ).contiguous().view(sz_b, len_q, -1)
 output = self.dropout(self.fc(output))
output = self.layer_norm(output + residual)
 return output, attn
class PositionwiseFeedForward(nn.Module):
def __init__(self, d_in, d_hid, dropout=0.1):
super().__init__()
self.w_1 = nn.Conv1d(d_in, d_hid, 1)
 self.w_2 = nn.Conv1d(d_hid, d_in, 1)
 self.layer_norm = nn.LayerNorm(d_in)
 self.dropout = nn.Dropout(dropout)
 def forward(self, x):
residual = x
output = x.transpose(1, 2)
 output = self.w_2(F.relu(self.w_1(output)))
 output = output.transpose(1, 2)
 output = self.dropout(output)
 output = self.layer_norm(output + residual)
 return output
class EncoderLayer(nn.Module):
def __init__(self, d_model, d_inner, n_head, d_k, d_v, dropout=0.1):
super(EncoderLayer, self ).__init__()
self.slf_attn = MultiHeadAttention(
 n_head, d_model, d_k, d_v, dropout=dropout)
 self.pos_ffn = PositionwiseFeedForward(d_model, d_inner, dropout=dropout)
 def forward(self, enc_input):
enc_output, enc_slf_attn = self.slf_attn(
 enc_input, enc_input, enc_input)
 enc_output = self.pos_ffn(enc_output)
 return enc_output, enc_slf_attn
class DecoderLayer(nn.Module):
def __init__(self, d_model, d_inner, n_head, d_k, d_v, dropout=0.1):
super(DecoderLayer, self ).__init__()
self.slf_attn = MultiHeadAttention(n_head, d_model, d_k, d_v, dropout=dropout)
 self.enc_attn = MultiHeadAttention(n_head, d_model, d_k, d_v, dropout=dropout)
 self.pos_ffn = PositionwiseFeedForward(d_model, d_inner, dropout=dropout)
 def forward(self, dec_input, enc_output, non_pad_mask=None, slf_attn_mask=None, dec_enc_attn_mask=None):
dec_output, dec_slf_attn = self.slf_attn(
 dec_input, dec_input, dec_input, mask=slf_attn_mask)
 dec_output, dec_enc_attn = self.enc_attn(
 dec_output, enc_output, enc_output, mask=dec_enc_attn_mask)
 dec_output = self.pos_ffn(dec_output)
 return dec_output, dec_slf_attn, dec_enc_attn
class Single_Global_SelfAttn_Module(nn.Module):
def __init__(
 self,
window, n_multiv, n_kernels, w_kernel,
d_k, d_v, d_model, d_inner,
n_layers, n_head, drop_prob=0.1):

 super(Single_Global_SelfAttn_Module, self ).__init__()
self.window = window
self.w_kernel = w_kernel
self.n_multiv = n_multiv
self.d_model = d_model
self.drop_prob = drop_prob
self.conv2 = nn.Conv2d(1, n_kernels,(window, w_kernel))
self.in_linear = nn.Linear(n_kernels, d_model)
 self.out_linear = nn.Linear(d_model, n_kernels)
 self.layer_stack = nn.ModuleList([
EncoderLayer(d_model, d_inner, n_head, d_k, d_v, dropout=drop_prob)
 for _ in range(n_layers)])
 def forward(self, x, return_attns=False):
x = x.view(-1, self.w_kernel, self.window, self.n_multiv)
 x2 = F.relu(self.conv2(x))
x2 = nn.Dropout(p=self.drop_prob )(x2)
 x = torch.squeeze(x2, 2)
 x = torch.transpose(x, 1, 2)
 src_seq = self.in_linear(x)
 enc_slf_attn_list = []
enc_output = src_seq
for enc_layer in self.layer_stack:
enc_output, enc_slf_attn = enc_layer(enc_output)
 if return_attns:
enc_slf_attn_list += [enc_slf_attn]
if return_attns:
return enc_output, enc_slf_attn_list
enc_output = self.out_linear(enc_output)
 return enc_output,
class Single_Local_SelfAttn_Module(nn.Module):
def __init__(
 self,
window, local, n_multiv, n_kernels, w_kernel,
d_k, d_v, d_model, d_inner,
n_layers, n_head, drop_prob=0.1):

 super(Single_Local_SelfAttn_Module, self ).__init__()
self.window = window
self.w_kernel = w_kernel
self.n_multiv = n_multiv
self.d_model = d_model
self.drop_prob = drop_prob
self.conv1 = nn.Conv2d(1, n_kernels,(local, w_kernel))
self.pooling1 = nn.AdaptiveMaxPool2d(( 1, n_multiv))
self.in_linear = nn.Linear(n_kernels, d_model)
 self.out_linear = nn.Linear(d_model, n_kernels)
 self.layer_stack = nn.ModuleList([
EncoderLayer(d_model, d_inner, n_head, d_k, d_v, dropout=drop_prob)
 for _ in range(n_layers)])
 def forward(self, x, return_attns=False):
x = x.view(-1, self.w_kernel, self.window, self.n_multiv)
 x1 = F.relu(self.conv1(x))
x1 = self.pooling1(x1)
 x1 = nn.Dropout(p=self.drop_prob )(x1)
 x = torch.squeeze(x1, 2)
 x = torch.transpose(x, 1, 2)
 src_seq = self.in_linear(x)
 enc_slf_attn_list = []
enc_output = src_seq
for enc_layer in self.layer_stack:
enc_output, enc_slf_attn = enc_layer(enc_output)
 if return_attns:
enc_slf_attn_list += [enc_slf_attn]
if return_attns:
return enc_output, enc_slf_attn_list
enc_output = self.out_linear(enc_output)
 return enc_output,
class AR(nn.Module):
def __init__(self, window):
super(AR, self ).__init__()
self.linear = nn.Linear(window, 1)
 def forward(self, x):
x = torch.transpose(x, 1, 2)
 x = self.linear(x)
 x = torch.transpose(x, 1, 2)
 return x
class DSANet(ptl.LightningModule):
def __init__(self, tra, validation, n_multiv, batch_size=16, window=64, local=3, n_kernels=32,
drop_prob=0.1, criterion='rmsle_loss', learning_rate=0.005, horizon=14):
super(DSANet, self ).__init__()
self.batch_size = batch_size
self.window = window
self.local = local
self.n_multiv = n_multiv
self.n_kernels = n_kernels
self.w_kernel = 1
self.d_model = 512
self.d_inner = 2048
self.n_layers = 6
self.n_head = 8
self.d_k = 64
self.d_v = 64
self.drop_prob = drop_prob
self.criterion = criterion
self.learning_rate = learning_rate
self.horizon = horizon
self.tra = tra
self.validation = validation
self.__build_model()
def __build_model(self):
self.sgsf = Single_Global_SelfAttn_Module(
 window=self.window, n_multiv=self.n_multiv, n_kernels=self.n_kernels,
w_kernel=self.w_kernel, d_k=self.d_k, d_v=self.d_v, d_model=self.d_model,
d_inner=self.d_inner, n_layers=self.n_layers, n_head=self.n_head, drop_prob=self.drop_prob)
 self.slsf = Single_Local_SelfAttn_Module(
 window=self.window, local=self.local, n_multiv=self.n_multiv, n_kernels=self.n_kernels,
w_kernel=self.w_kernel, d_k=self.d_k, d_v=self.d_v, d_model=self.d_model,
d_inner=self.d_inner, n_layers=self.n_layers, n_head=self.n_head, drop_prob=self.drop_prob)
 self.ar = AR(window=self.window)
 self.W_output1 = nn.Linear(2 * self.n_kernels, 1)
 self.dropout = nn.Dropout(p=self.drop_prob)
 self.active_func = nn.Tanh()
def forward(self, x):
sgsf_output, *_ = self.sgsf(x)
 slsf_output, *_ = self.slsf(x)
 sf_output = torch.cat(( sgsf_output, slsf_output), 2)
 sf_output = self.dropout(sf_output)
 sf_output = self.W_output1(sf_output)
 sf_output = torch.transpose(sf_output, 1, 2)
 ar_output = self.ar(x)
 output = sf_output + ar_output
output[output < 0] = 0.0
return output
def loss(self, labels, predictions):
if self.criterion == 'l1_loss':
loss = F.l1_loss(predictions, labels)
 elif self.criterion == 'mse_loss':
loss = F.mse_loss(predictions, labels)
 elif self.criterion == 'rmsle_loss':
loss = rmsle(predictions, labels)
 return loss
def training_step(self, data_batch, batch_i):
x, y = data_batch
y_hat = self.forward(x)
 loss_val = self.loss(y, y_hat)
 if self.trainer.use_dp:
loss_val = loss_val.unsqueeze(0)
 output = OrderedDict({
'loss': loss_val
})
 return output
def validation_step(self, data_batch, batch_i):
x, y = data_batch
y_hat = self.forward(x)
 loss_val = self.loss(y, y_hat)
 if self.trainer.use_dp:
loss_val = loss_val.unsqueeze(0)
 output = OrderedDict({
'val_loss': loss_val,
'y': y,
'y_hat': y_hat,
})
 return output
def validation_epoch_end(self, outputs):
loss_sum = 0
for x in outputs:
loss_sum += x['val_loss'].item()
val_loss_mean = loss_sum / len(outputs)
 y = torch.cat(( [x['y'] for x in outputs]), 0)
 y_hat = torch.cat(( [x['y_hat'] for x in outputs]), 0)
 num_var = y.size(-1)
 y = y.view(-1, num_var)
 y_hat = y_hat.view(-1, num_var)
 sample_num = y.size(0)
 y_diff = y_hat - y
y_mean = torch.mean(y)
 y_translation = y - y_mean
val_rrse = torch.sqrt(torch.sum(torch.pow(y_diff, 2)))/ torch.sqrt(torch.sum(torch.pow(y_translation, 2)))
 y_m = torch.mean(y, 0, True)
 y_hat_m = torch.mean(y_hat, 0, True)
 y_d = y - y_m
y_hat_d = y_hat - y_hat_m
corr_top = torch.sum(y_d * y_hat_d, 0)
 corr_bottom = torch.sqrt(( torch.sum(torch.pow(y_d, 2), 0)* torch.sum(torch.pow(y_hat_d, 2), 0)))
 corr_inter = corr_top / corr_bottom
val_corr =(1./ num_var)* torch.sum(corr_inter)
 val_mae =(1./(sample_num * num_var)) * torch.sum(torch.abs(y_diff))
tqdm_dic = {
'val_loss': val_loss_mean,
'RRSE': val_rrse.item() ,
'CORR': val_corr.item() ,
'MAE': val_mae.item()
}
return tqdm_dic
def configure_optimizers(self):
optimizer = optim.Adam(self.parameters() , lr=self.learning_rate)
 scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)
 return [optimizer], [scheduler]
def __dataloader(self, train):
set_type = train
dataset = MTSFDataset(window=self.window, horizon=self.horizon,
set_type=set_type,
tra=self.tra, validation=self.validation)
 train_sampler = None
batch_size = self.batch_size
should_shuffle = train_sampler is None
loader = DataLoader(
 dataset=dataset,
batch_size=batch_size,
shuffle=should_shuffle,
sampler=train_sampler,
num_workers=4 
)
 return loader
@ptl.data_loader
def train_dataloader(self):
return self.__dataloader(train='train')
 @ptl.data_loader
def val_dataloader(self):
return self.__dataloader(train='validation' )<train_model>
train_features = pd.concat(( train_features, train_features_cluster,train_cluster_pca,train_features_stats), axis=1)
 test_features = pd.concat(( test_features, test_features_cluster,test_cluster_pca,test_features_stats), axis=1 )
model_cases = DSANet(np.array([train_fatal, train_cases]), np.array([valid_fatal, valid_cases]), train_cases.shape[1], window=win,
learning_rate=0.01, horizon=hor, drop_prob=0.5, batch_size=128)
 trainer = ptl.Trainer(val_check_interval=1, max_steps=5000, gpus=1, show_progress_bar=False)
 trainer.fit(model_cases )<data_type_conversions>
cols2 = train_targets_nonscored.columns.to_list()
uniques2 = {col: train_targets_nonscored[col].nunique() for col in cols2}
uniques2=pd.DataFrame(uniques2, index=[0] ).T
uniques2=uniques2.rename(columns={0:'count'})
 uniques2= uniques2.drop('sig_id', axis=0)
 print(f"{len(uniques2[uniques2['count']==1])} targets without ANY mechanism of action in the nonscored dataset" )
pred_size =(np.array(test_last_date, dtype='datetime64[D]' ).astype(np.int64)- np.array(test_first_date, dtype='datetime64[D]' ).astype(np.int64)) +1<load_from_csv>
nonmoacols=uniques2[uniques2['count']==1].index
train_targets_nonscored_columns = [col for col in list(train_targets_nonscored.columns)if col not in nonmoacols]
train_targets_nonscored=train_targets_nonscored[train_targets_nonscored_columns]
sub = pd.read_csv("/kaggle/input/covid-sberbank/sample_submission.csv")
 sub<count_values>
train = train_features.merge(train_targets_nonscored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_nonscored.columns]
sub.country.value_counts().sort_values()<feature_engineering>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
sub = pd.read_csv(f"/kaggle/input/covid19-global-forecasting-week-{week}/submission.csv")
 sub = test.merge(res, how='left', on=['Date', 'Province_State', 'Country_Region'] ).loc[:, ["ForecastId", "ConfirmedCases", "Fatalities"]]
sub['Fatalities'] = np.array(sub['Fatalities'], dtype=np.int)
 sub["ConfirmedCases"] = np.array(sub["ConfirmedCases"], dtype=np.int)
 sub<save_to_csv>
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
sub.to_csv("submission.csv", index=False )<set_options>
train = pd.get_dummies(train, columns=['cp_time','cp_dose'])
 test_ = pd.get_dummies(test, columns=['cp_time','cp_dose'] )
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
pd.set_option('display.max_columns', 99)
 pd.set_option('display.max_rows', 99)
<set_options>
feature_cols = [c for c in train.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['sig_id']]
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
sns.set_palette(sns.color_palette('tab20', 20)) 
<load_from_csv>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
DATA_PATH = '.. /input/covid19-metadata'
DATEFORMAT = '%Y-%m-%d'
def get_comp_data(COMP):
train = pd.read_csv(f'{COMP}/train.csv')
 test = pd.read_csv(f'{COMP}/test.csv')
 submission = pd.read_csv(f'{COMP}/submission.csv')
 print(train.shape, test.shape, submission.shape)
 train['Country_Region'] = train['Country_Region'].str.replace(',', '')
 test['Country_Region'] = test['Country_Region'].str.replace(',', '')
 train['Location'] = train['Country_Region'] + '-' + train['Province_State'].fillna('')
 test['Location'] = test['Country_Region'] + '-' + test['Province_State'].fillna('')
 train['LogConfirmed'] = to_log(train.ConfirmedCases)
 train['LogFatalities'] = to_log(train.Fatalities)
 train = train.drop(columns=['Province_State'])
 test = test.drop(columns=['Province_State'])
 country_codes = pd.read_csv(f'{DATA_PATH}/country_codes.csv', keep_default_na=False)
 train = train.merge(country_codes, on='Country_Region', how='left')
 test = test.merge(country_codes, on='Country_Region', how='left')
 train['DateTime'] = pd.to_datetime(train['Date'])
 test['DateTime'] = pd.to_datetime(test['Date'])
 train = train.sort_values(by='Date')
 test = test.sort_values(by='Date')
 train = train.fillna('
test = test.fillna('
return train, test, submission
def process_each_location(df):
dfs = []
for loc, df in df.groupby('Location'):
df = df.sort_values(by='Date')
 df['Fatalities'] = df['Fatalities'].cummax()
df['ConfirmedCases'] = df['ConfirmedCases'].cummax()
df['LogFatalities'] = df['LogFatalities'].cummax()
df['LogConfirmed'] = df['LogConfirmed'].cummax()
df['LogConfirmedNextDay'] = df['LogConfirmed'].shift(-1)
 df['ConfirmedNextDay'] = df['ConfirmedCases'].shift(-1)
 df['DateNextDay'] = df['Date'].shift(-1)
 df['LogFatalitiesNextDay'] = df['LogFatalities'].shift(-1)
 df['FatalitiesNextDay'] = df['Fatalities'].shift(-1)
 df['LogConfirmedDelta'] = df['LogConfirmedNextDay'] - df['LogConfirmed']
df['ConfirmedDelta'] = df['ConfirmedNextDay'] - df['ConfirmedCases']
df['LogFatalitiesDelta'] = df['LogFatalitiesNextDay'] - df['LogFatalities']
df['FatalitiesDelta'] = df['FatalitiesNextDay'] - df['Fatalities']
dfs.append(df)
 return pd.concat(dfs)
 def add_days(d, k):
return dt.datetime.strptime(d, DATEFORMAT)+ dt.timedelta(days=k)
 def to_log(x):
return np.log(x + 1)
 def to_exp(x):
return np.exp(x)- 1
def create_features(train_set):
dfs = []
for loc, df in train_set.groupby('Location'):
df = df.sort_values(by='Date' ).copy()
df['f_lc_7d'] = df['LogConfirmed'].shift(7)
 df['f_lf_7d'] = df['LogFatalities'].shift(7)
 df['f_lc_3d'] = df['LogConfirmed'].shift(3)
 df['f_lf_3d'] = df['LogFatalities'].shift(3)
 df['f_lc_1d'] = df['LogConfirmed'].shift(1)
 df['f_lf_1d'] = df['LogFatalities'].shift(1)
 df['f_lc_0d'] = df['LogConfirmed']
df['f_lf_0d'] = df['LogFatalities']
df['f_fc_rate'] = np.clip(( to_exp(df['LogFatalities'])+ 1)/(to_exp(df['LogConfirmed'])+ 1), 0, 0.15)
 dfs.append(df)
 dfs = pd.concat(dfs)
 dfs['d_lc_7d'] = dfs['f_lc_0d'] - dfs['f_lc_7d']
dfs['d_lf_7d'] = dfs['f_lf_0d'] - dfs['f_lf_7d']
dfs['d_lc_3d'] = dfs['f_lc_3d'] - dfs['f_lc_3d']
dfs['d_lf_3d'] = dfs['f_lf_3d'] - dfs['f_lf_3d']
dfs['d_lc_1d'] = dfs['f_lc_0d'] - dfs['f_lc_1d']
dfs['d_lf_1d'] = dfs['f_lf_0d'] - dfs['f_lf_1d']
return dfs 
<feature_engineering>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
COMP = '.. /input/covid19-global-forecasting-week-4'
start = dt.datetime.now()
train, test, submission = get_comp_data(COMP)
 train.shape, test.shape, submission.shape
train.head(2)
 test.head(2)
 TRAIN_START = train.Date.min()
TEST_START = test.Date.min()
TRAIN_END = train.Date.max()
TEST_END = test.Date.max()
print(TRAIN_START, TRAIN_END, TEST_START, TEST_END )<categorify>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
train_clean = process_each_location(train)
 print('train cleaned', train_clean.shape)
 train_clean = train_clean[[
'Location', 'Date', 'continent',
'LogConfirmed', 'LogFatalities',
'LogConfirmedDelta', 'LogFatalitiesDelta'
]]
continent_encoder = preprocessing.LabelEncoder()
train_clean['f_continent'] = continent_encoder.fit_transform(train_clean.continent)
 train_features = create_features(train_clean)
 train_features.head()<categorify>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.leaky_relu(self.dense1(x), 1e-3)
 x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
def predict(min_child_weight, eta, colsample_bytree, max_depth, subsample,
NROUND, PRECISION, DECAY, WEIGHT_NORM, MIN_DATE):
train, test, submission = get_comp_data(COMP)
 train_clean = process_each_location(train)
 print('train cleaned', train_clean.shape)
 train_clean = train_clean[[
'Location', 'Date', 'continent',
'LogConfirmed', 'LogFatalities',
'LogConfirmedDelta', 'LogFatalitiesDelta'
]]
continent_encoder = preprocessing.LabelEncoder()
train_clean['f_continent'] = continent_encoder.fit_transform(train_clean.continent)
 train_features = create_features(train_clean)
 VALID_CUTOFF = TRAIN_END
features = [
'f_lc_7d', 'f_lf_7d', 'f_lc_3d', 'f_lf_3d', 'f_continent',
'f_lc_1d', 'f_lf_1d', 'f_lc_0d', 'f_lf_0d', 'f_fc_rate'
]
diff_features = ['d_lc_7d', 'd_lf_7d', 'd_lc_3d', 'd_lf_3d', 'd_lc_1d', 'd_lf_1d']
features = features + diff_features
print(f'{len(features)} features: ')
 print(features)
 fix = {
'lambda': 1., 'nthread': 3, 'booster': 'gbtree',
'silent': 1, 'eval_metric': 'rmse',
'objective': 'reg:squarederror'}
config = dict(
 min_child_weight=min_child_weight,
eta=eta, colsample_bytree=colsample_bytree,
max_depth=max_depth, subsample=subsample)
 config.update(fix)
 Xtr = train_features[(train_features.Date >= MIN_DATE)&(train_features.Date < VALID_CUTOFF)].copy()
Xtr['days'] = -(pd.to_datetime(train_features.Date)- dt.datetime.strptime(VALID_CUTOFF, DATEFORMAT)).dt.days
print(Xtr.shape)
 print(config)
 print(PRECISION, NROUND, DECAY, WEIGHT_NORM)
 def weighting(days):
return 1./ days ** WEIGHT_NORM
dtrain_lc = xgb.DMatrix(Xtr[features].round(PRECISION), label=Xtr.LogConfirmedDelta, weight=weighting(Xtr.days))
dtrain_lf = xgb.DMatrix(Xtr[features].round(PRECISION), label=Xtr.LogFatalitiesDelta, weight=weighting(Xtr.days))
model_lc = xgb.train(config, dtrain_lc, NROUND, evals=[(dtrain_lc, 'train-lc')], verbose_eval=100)
 model_lf = xgb.train(config, dtrain_lf, NROUND, evals=[(dtrain_lf, 'train-lf')], verbose_eval=100)
 predictions = Xtr.copy()
predictions = train_features[(train_features.Date >= MIN_DATE)&(train_features.Date <= VALID_CUTOFF)].copy()
predictions.LogConfirmedDelta = np.nan
predictions.LogFatalitiesDelta = np.nan
for i, d in enumerate(pd.date_range(VALID_CUTOFF, add_days(TEST_END, 1))):
last_day = str(d ).split(' ')[0]
next_day = dt.datetime.strptime(last_day, DATEFORMAT)+ dt.timedelta(days=1)
 next_day = next_day.strftime(DATEFORMAT)
 p_next_day = predictions[predictions.Date == last_day].copy()
p_next_day.Date = next_day
p_next_day['plc'] = model_lc.predict(xgb.DMatrix(p_next_day[features].round(PRECISION)))
 p_next_day['plf'] = model_lf.predict(xgb.DMatrix(p_next_day[features].round(PRECISION)))
 p_next_day.LogConfirmed = p_next_day.LogConfirmed + np.clip(p_next_day['plc'], 0, None)* DECAY ** i
p_next_day.LogFatalities = p_next_day.LogFatalities + np.clip(p_next_day['plf'], 0, None)* DECAY ** i
predictions = pd.concat([predictions, p_next_day], sort=True)
 predictions = create_features(predictions)
 predictions['PC'] = to_exp(predictions.LogConfirmed)
 predictions['PF'] = to_exp(predictions.LogFatalities)
 return predictions<predict_on_test>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 26
BATCH_SIZE = 256
LEARNING_RATE = 6e-4
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048
decay = 0.99
prediction_1 = predict(min_child_weight=5, eta=0.01, colsample_bytree=0.8, max_depth=5, subsample=0.9,
NROUND=800, PRECISION=2, DECAY=decay, WEIGHT_NORM=0.25, MIN_DATE='2020-03-22')
 prediction_2 = predict(min_child_weight=7, eta=0.01, colsample_bytree=0.7, max_depth=6, subsample=0.8,
NROUND=1000, PRECISION=2, DECAY=decay, WEIGHT_NORM=0.23, MIN_DATE='2020-03-15')
 prediction_3 = predict(min_child_weight=3, eta=0.01, colsample_bytree=0.6, max_depth=7, subsample=0.7,
NROUND=1200, PRECISION=2, DECAY=decay, WEIGHT_NORM=0.2, MIN_DATE='2020-03-08')
 prediction_4 = predict(min_child_weight=3, eta=0.011, colsample_bytree=0.75, max_depth=10, subsample=0.6,
NROUND=1200, PRECISION=3, DECAY=decay, WEIGHT_NORM=0.15, MIN_DATE='2020-03-15')
 prediction_5 = predict(min_child_weight=10, eta=0.008, colsample_bytree=0.75, max_depth=10, subsample=0.6,
NROUND=1500, PRECISION=3, DECAY=decay, WEIGHT_NORM=0.2, MIN_DATE='2020-03-22')
 prediction_6 = predict(min_child_weight=20, eta=0.01, colsample_bytree=0.7, max_depth=5, subsample=0.85,
NROUND=1000, PRECISION=3, DECAY=decay, WEIGHT_NORM=0.3, MIN_DATE='2020-03-22' )<merge>
def run_training(fold, seed):
seed_everything(seed)
 mskf = MultilabelStratifiedKFold(n_splits=7,random_state=seed)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
train.loc[v_idx, 'kfold'] = int(f)
 train['kfold'] = train['kfold'].astype(int)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"SEED: {seed}, FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed} ,FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"SEED{seed}_FOLD{fold}_nonscored.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"SEED{seed}_FOLD{fold}_nonscored.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
cols = ['Location', 'Date', 'PC', 'PF']
p12 = pd.merge(prediction_1[cols], prediction_2[cols], on=['Location', 'Date'], suffixes=['_1', '_2'])
 p34 = pd.merge(prediction_3[cols], prediction_4[cols], on=['Location', 'Date'], suffixes=['_3', '_4'])
 p56 = pd.merge(prediction_5[cols], prediction_6[cols], on=['Location', 'Date'], suffixes=['_5', '_6'])
 preds = pd.merge(p12, p34, on=['Location', 'Date'])
 preds = pd.merge(preds, p56, on=['Location', 'Date'])
 preds.head()
c = preds.loc[preds.Date >= '2020-04-15'].corr()
c
fig = px.imshow(c)
 fig.show()<feature_engineering>
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
pcs = ['PC_1', 'PC_2', 'PC_3', 'PC_4', 'PC_5', 'PC_6']
pfs = ['PF_1', 'PF_2', 'PF_3', 'PF_4', 'PF_5', 'PF_6']
preds['PC'] = to_exp(to_log(preds[pcs] ).mean(axis=1))
preds['PF'] = to_exp(to_log(preds[pfs] ).mean(axis=1))
preds.tail()<merge>
SEED = [0,1,2,3,4,5,6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test_[target_cols] = predictions
my_submission = test.copy()
my_submission = my_submission.merge(preds)
 my_submission['ConfirmedCases'] = my_submission['PC']
my_submission['Fatalities'] = my_submission['PF']
my_submission.shape
my_submission.head()<save_to_csv>
train = train.merge(train_targets_scored, on='sig_id')
 target = train[train_targets_scored.columns]
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
my_submission[[
'ForecastId', 'ConfirmedCases', 'Fatalities'
]].to_csv('submission.csv', index=False)
 my_submission.groupby('Date')[['ConfirmedCases', 'Fatalities']].sum().reset_index().tail()<train_model>
feature_cols = [c for c in train.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['sig_id','kfold']]
end = dt.datetime.now()
print('Finished', end,(end - start ).seconds, 's' )<install_modules>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 26
BATCH_SIZE = 256
LEARNING_RATE = 6e-4
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048
!pip install fastai2
!pip install fast_tabnet<load_from_csv>
def run_training(fold, seed):
seed_everything(seed)
 mskf = MultilabelStratifiedKFold(n_splits=7,random_state=seed)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
train.loc[v_idx, 'kfold'] = int(f)
 train['kfold'] = train['kfold'].astype(int)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"SEED: {seed}, FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed} ,FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"SEED{seed}_FOLD{fold}_scored.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"SEED{seed}_FOLD{fold}_scored.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
PATH = '/kaggle/input/covid19-global-forecasting-week-4/'
train_df = pd.read_csv(PATH + 'train.csv', parse_dates=['Date'])
 test_df = pd.read_csv(PATH + 'test.csv', parse_dates=['Date'])
 add_datepart(train_df, 'Date', drop=False)
 add_datepart(test_df, 'Date', drop=False)
 train_df.shape<load_from_csv>
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
PATH1 = '/kaggle/input/covid19-country-data-wk3-release/'
meta_convert_fun = lambda x: np.float32(x)if x not in ['N.A.', '
meta_df = pd.read_csv(PATH1 + 'Data Join - RELEASE.csv', thousands=",",
converters={
' TFR ': meta_convert_fun,
'Personality_uai': meta_convert_fun,
} ).rename(columns=lambda x: x.strip())
 PATH2 = '/kaggle/input/countryinfo/'
countryinfo = pd.read_csv(PATH2 + 'covid19countryinfo.csv', thousands=",", parse_dates=['quarantine', 'schools', 'publicplace', 'gathering', 'nonessential'])
 testinfo = pd.read_csv(PATH2 + 'covid19tests.csv', thousands=",")
 countryinfo.rename(columns={'region': 'Province_State', 'country': 'Country_Region'}, inplace=True)
 testinfo.rename(columns={'region': 'Province_State', 'country': 'Country_Region'}, inplace=True)
 testinfo = testinfo.drop(['alpha3code', 'alpha2code', 'date'], axis=1)
 PATH3 = '/kaggle/input/covid19-forecasting-metadata/'
continent_meta = pd.read_csv(PATH3 + 'region_metadata.csv' ).rename(columns={'density': 'pop_density'})
 continent_meta = continent_meta[['Country_Region' ,'Province_State', 'continent', 'lat', 'lon', 'pop_density']]
recoveries_meta = pd.read_csv(PATH3 + 'region_date_metadata.csv', parse_dates=['Date'])
 def fill_unknown_state(df):
df.fillna({'Province_State': 'Unknown'}, inplace=True)
 for d in [train_df, test_df, meta_df, countryinfo, testinfo, continent_meta, recoveries_meta]:
fill_unknown_state(d )<filter>
SEED = [0,1,2,3,4,5,6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions
outliars = ['China']
out_inputs = {}
for out in outliars:
out_inputs[out] = train_df[train_df['Country_Region'] == out]
train_df.drop(train_df.index[train_df['Country_Region'] == out], inplace=True )<merge>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
cv_score = 0
roc_score = 0
for i in range(len(target_cols)) :
cv_score += log_loss(y_true[:, i], y_pred[:, i])
 roc_score +=roc_auc_score(y_true[:, i], y_pred[:, i], average='micro')
 print("CV log_loss: ", cv_score / y_pred.shape[1])
 print("Overall AUC: ", roc_score / y_pred.shape[1] )
test_ori = test_df.copy()
merge_cols = ['Province_State', 'Country_Region', 'Date']
test_hlp = test_df[merge_cols + ['ForecastId']]
fst_date = test_hlp.Date.min()
outlier_dfs = []
for out, in_df in out_inputs.items() :
last_date = in_df.Date.max()
merged = in_df[in_df['Date'] >= fst_date].merge(test_hlp, on=merge_cols, how='left')[['ForecastId', 'ConfirmedCases', 'Fatalities']]
future_test = test_hlp[(test_hlp['Country_Region'] == out)&(test_hlp['Date'] > last_date)]
to_add = in_df.groupby(['Province_State', 'Country_Region'] ).last().reset_index() [['Province_State', 'Country_Region', 'ConfirmedCases', 'Fatalities']]
merged_future = future_test.merge(to_add, on=['Province_State', 'Country_Region'], how='left')[['ForecastId', 'ConfirmedCases', 'Fatalities']]
merged = pd.concat([merged, merged_future], sort=True)
 test_df.drop(test_df[test_df['ForecastId'].isin(merged.ForecastId)].index, inplace=True)
 outlier_dfs.append(merged)
 outlier_df = pd.concat(outlier_dfs, sort=True )<drop_column>
oof_pretrain= valid_results[target_cols]
oof_pretrain.to_csv('off_pretrain_last.csv', index=False )
<merge><EOS> 
public_id = list(df['sig_id'].values)
 test_id = list(test_features['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=target_cols)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_submit.loc[test.sig_id,:] = test[target_cols].values
df_submit.loc[test_features[test_features.cp_type=='ctl_vehicle'].sig_id]= 0
df_submit.to_csv('submission.csv',index=True )