Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<categorify>
warnings.filterwarnings("ignore" )
def _convert_to_transformer_inputs(title, question, answer, tokenizer, max_sequence_length):

 def return_id(str1, str2, truncation_strategy, length):
inputs = tokenizer.encode_plus(str1, str2,
add_special_tokens=True,
max_length=length,
truncation_strategy=truncation_strategy)
 input_ids = inputs["input_ids"]
input_masks = [1] * len(input_ids)
 input_segments = inputs["token_type_ids"]
padding_length = length - len(input_ids)
 padding_id = tokenizer.pad_token_id
input_ids = input_ids +([padding_id] * padding_length)
 input_masks = input_masks +([0] * padding_length)
 input_segments = input_segments +([0] * padding_length)
 return [input_ids, input_masks, input_segments]
input_ids_q, input_masks_q, input_segments_q = return_id(
 title + ' ' + question, None, 'longest_first', max_sequence_length)
 input_ids_a, input_masks_a, input_segments_a = return_id(
 answer, None, 'longest_first', max_sequence_length)
 return [input_ids_q, input_masks_q, input_segments_q,
input_ids_a, input_masks_a, input_segments_a]
def compute_input_arrays(df, columns, tokenizer, max_sequence_length):
input_ids_q, input_masks_q, input_segments_q = [], [], []
input_ids_a, input_masks_a, input_segments_a = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
ids_q, masks_q, segments_q, ids_a, masks_a, segments_a = \
_convert_to_transformer_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids_q.append(ids_q)
 input_masks_q.append(masks_q)
 input_segments_q.append(segments_q)
 input_ids_a.append(ids_a)
 input_masks_a.append(masks_a)
 input_segments_a.append(segments_a)
 return [np.asarray(input_ids_q, dtype=np.int32),
np.asarray(input_masks_q, dtype=np.int32),
np.asarray(input_segments_q, dtype=np.int32),
np.asarray(input_ids_a, dtype=np.int32),
np.asarray(input_masks_a, dtype=np.int32),
np.asarray(input_segments_a, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )<feature_engineering>
train = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 submission = pd.read_csv(".. /input/covid19-global-forecasting-week-4/submission.csv" )
def compute_spearmanr_ignore_nan(trues, preds):
rhos = []
for tcol, pcol in zip(np.transpose(trues), np.transpose(preds)) :
rhos.append(spearmanr(tcol, pcol ).correlation)
 return np.nanmean(rhos)
 def create_model() :
q_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 config = BertConfig()
config.output_hidden_states = False
bert_model = TFBertModel.from_pretrained(
 BERT_PATH+'bert-base-uncased-tf_model.h5', config=config)
 q_embedding = bert_model(q_id, attention_mask=q_mask, token_type_ids=q_atn)[0]
a_embedding = bert_model(a_id, attention_mask=a_mask, token_type_ids=a_atn)[0]
q = tf.keras.layers.GlobalAveragePooling1D()(q_embedding)
 a = tf.keras.layers.GlobalAveragePooling1D()(a_embedding)
 x = tf.keras.layers.Concatenate()([q, a])
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(30, activation='sigmoid' )(x)
 model = tf.keras.models.Model(inputs=[q_id, q_mask, q_atn, a_id, a_mask, a_atn,], outputs=x)
 return model<categorify>
train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'] )
outputs = compute_output_arrays(df_train, output_categories)
 inputs = compute_input_arrays(df_train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arrays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
<split>
train = train[train['Date']<='2020-04-14']
train['part'] = 'train'
test['part'] = 'test'
gkf = GroupKFold(n_splits=5 ).split(X=df_train.question_body, groups=df_train.question_body)
 valid_preds = []
test_preds = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold in [0, 2]:
train_inputs = [inputs[i][train_idx] for i in range(len(inputs)) ]
train_outputs = outputs[train_idx]
valid_inputs = [inputs[i][valid_idx] for i in range(len(inputs)) ]
valid_outputs = outputs[valid_idx]
K.clear_session()
model = create_model()
optimizer = tf.keras.optimizers.Adam(learning_rate=2e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 model.fit(train_inputs, train_outputs, epochs=3, batch_size=6)
 valid_preds.append(model.predict(valid_inputs))
test_preds.append(model.predict(test_inputs))
rho_val = compute_spearmanr_ignore_nan(valid_outputs, valid_preds[-1])
 print('validation score = ', rho_val )<save_to_csv>
train['Location'] = train['Province_State'].astype(str)+ train['Country_Region'].astype(str)
 test['Location'] = test['Province_State'].astype(str)+ test['Country_Region'].astype(str )
df_sub.iloc[:, 1:] = np.average(test_preds, axis=0)
 df_sub.to_csv('submission.csv', index=False )<set_options>
test = test.merge(train[['ConfirmedCases','Fatalities','Location','Date']],how='left',on=['Location','Date'] )
np.set_printoptions(suppress=True)
 print(tf.__version__ )<load_from_csv>
train = train[train['Date']<='2020-04-01']
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-uncased-huggingface-transformer/'
tokenizer = BertTokenizer.from_pretrained(BERT_PATH+'bert-base-uncased-vocab.txt')
 MAX_SEQUENCE_LENGTH = 384
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )<categorify>
data = pd.concat([train,test],axis=0)
 data = data.sort_values(['Country_Region','Date'] )
def _convert_to_transformer_inputs(title, question, answer, tokenizer, max_sequence_length):

 def return_id(str1, str2, truncation_strategy, length):
inputs = tokenizer.encode_plus(str1, str2,
add_special_tokens=True,
max_length=length,
truncation_strategy=truncation_strategy)
 input_ids = inputs["input_ids"]
input_masks = [1] * len(input_ids)
 input_segments = inputs["token_type_ids"]
padding_length = length - len(input_ids)
 padding_id = tokenizer.pad_token_id
input_ids = input_ids +([padding_id] * padding_length)
 input_masks = input_masks +([0] * padding_length)
 input_segments = input_segments +([0] * padding_length)
 return [input_ids, input_masks, input_segments]
input_ids_q, input_masks_q, input_segments_q = return_id(
 title + ' ' + question, None, 'longest_first', max_sequence_length)
 input_ids_a, input_masks_a, input_segments_a = return_id(
 answer, None, 'longest_first', max_sequence_length)
 return [input_ids_q, input_masks_q, input_segments_q,
input_ids_a, input_masks_a, input_segments_a]
def compute_input_arrays(df, columns, tokenizer, max_sequence_length):
input_ids_q, input_masks_q, input_segments_q = [], [], []
input_ids_a, input_masks_a, input_segments_a = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
ids_q, masks_q, segments_q, ids_a, masks_a, segments_a = \
_convert_to_transformer_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids_q.append(ids_q)
 input_masks_q.append(masks_q)
 input_segments_q.append(segments_q)
 input_ids_a.append(ids_a)
 input_masks_a.append(masks_a)
 input_segments_a.append(segments_a)
 return [np.asarray(input_ids_q, dtype=np.int32),
np.asarray(input_masks_q, dtype=np.int32),
np.asarray(input_segments_q, dtype=np.int32),
np.asarray(input_ids_a, dtype=np.int32),
np.asarray(input_masks_a, dtype=np.int32),
np.asarray(input_segments_a, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )<feature_engineering>
data = data.melt(id_vars=['ForecastId','Date','Location','part'],value_vars=['ConfirmedCases','Fatalities'],value_name='Target' ).sort_values(['Location','Date'] )
def compute_spearmanr_ignore_nan(trues, preds):
rhos = []
for tcol, pcol in zip(np.transpose(trues), np.transpose(preds)) :
rhos.append(spearmanr(tcol, pcol ).correlation)
 return np.nanmean(rhos)
 def create_model() :
q_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_id = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_mask = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 q_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 a_atn = tf.keras.layers.Input(( MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
 config = BertConfig()
config.output_hidden_states = False
bert_model = TFBertModel.from_pretrained(
 BERT_PATH+'bert-base-uncased-tf_model.h5', config=config)
 q_embedding = bert_model(q_id, attention_mask=q_mask, token_type_ids=q_atn)[0]
a_embedding = bert_model(a_id, attention_mask=a_mask, token_type_ids=a_atn)[0]
q = tf.keras.layers.GlobalAveragePooling1D()(q_embedding)
 a = tf.keras.layers.GlobalAveragePooling1D()(a_embedding)
 x = tf.keras.layers.Concatenate()([q, a])
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(30, activation='sigmoid' )(x)
 model = tf.keras.models.Model(inputs=[q_id, q_mask, q_atn, a_id, a_mask, a_atn,], outputs=x)
 return model<categorify>
data['Day'] = data['Date'].astype(str ).apply(lambda x: int(''.join(x.split('-')[1:])))
 data['Month'] = data.Date.dt.month
outputs = compute_output_arrays(df_train, output_categories)
 inputs = compute_input_arrays(df_train, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
 test_inputs = compute_input_arrays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH)
<split>
data['lag_1'] = data.groupby(['Location','variable'])['Target'].transform(lambda x: x.shift(1))
data['lag_2'] = data.groupby(['Location','variable'])['Target'].transform(lambda x: x.shift(2))
data['lag_3'] = data.groupby(['Location','variable'])['Target'].transform(lambda x: x.shift(3))
data['lag_4'] = data.groupby(['Location','variable'])['Target'].transform(lambda x: x.shift(4))
gkf = GroupKFold(n_splits=5 ).split(X=df_train.question_body, groups=df_train.question_body)
 valid_preds = []
test_preds = []
for fold,(train_idx, valid_idx)in enumerate(gkf):
if fold in [0, 2]:
train_inputs = [inputs[i][train_idx] for i in range(len(inputs)) ]
train_outputs = outputs[train_idx]
valid_inputs = [inputs[i][valid_idx] for i in range(len(inputs)) ]
valid_outputs = outputs[valid_idx]
K.clear_session()
model = create_model()
optimizer = tf.keras.optimizers.Adam(learning_rate=2e-5)
 model.compile(loss='binary_crossentropy', optimizer=optimizer)
 model.fit(train_inputs, train_outputs, epochs=3, batch_size=6)
 valid_preds.append(model.predict(valid_inputs))
test_preds.append(model.predict(test_inputs))
rho_val = compute_spearmanr_ignore_nan(valid_outputs, valid_preds[-1])
 print('validation score = ', rho_val )<save_to_csv>
data['Diff1'] = data['lag_1'] - data['lag_2']
data['Diff2'] = data['lag_2'] - data['lag_3']
data['Diff3'] = data['lag_3'] - data['lag_4']
data['Diffavg'] =(data['Diff1'] + data['Diff2'] +data['Diff3'])/3
df_sub.iloc[:, 1:] = np.average(test_preds, axis=0)
 df_sub.to_csv('submission.csv', index=False )<import_modules>
data["Inc1"] =(data['Diff1'] / data['lag_2'])*100
data["Inc2"] =(data['Diff2'] / data['lag_3'])*100
data["Inc3"] =(data['Diff3'] / data['lag_4'])*100
data['Incavg'] =(data['Inc1'] + data['Inc2'] +data['Inc3'])/3
from __future__ import print_function, division
import os
import torch
import torch.nn as nn
import torch.optim as optim
import pandas as pd
from skimage import io
import numpy as np
from torch.utils.data import Dataset, DataLoader, random_split
from torchvision import transforms
import matplotlib.pyplot as plt<load_from_csv>
data = data[data['Date']>'2020-02-19']
data.drop(['Diff1','Diff2','Diff3'],axis=1,inplace=True )
class CactusImageDataset(Dataset):
"Aerial Cactus Classification Dataset"
def __init__(self, csv_file, root_dir, transform = None):

 self.cactus_annotations = pd.read_csv(csv_file)
 self.root_dir = root_dir
self.transform = transform
def __len__(self):
return self.cactus_annotations.shape[0]
def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
img_name = os.path.join(self.root_dir,
self.cactus_annotations.iloc[idx, 0])
 image = io.imread(img_name)
 has_cactus = self.cactus_annotations.iloc[idx, 1]
if self.transform:
image = self.transform(image)
 sample = {"image":image, 'label':has_cactus}
return sample 
<categorify>
encoderloc = LabelEncoder()
encodervar = LabelEncoder()
data['Location'] = encoderloc.fit_transform(data['Location'])
 data['variable'] = encodervar.fit_transform(data['variable'])
 data.head()
image_transforms = {
'train': transforms.Compose([
transforms.ToPILImage() ,
transforms.RandomHorizontalFlip() ,
transforms.RandomVerticalFlip() ,
transforms.ToTensor() ,
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
 ]),
'test':transforms.Compose([
transforms.ToPILImage() ,
transforms.ToTensor() ,
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
 ])
 } 
<load_from_csv>
features = ['Day','Location','variable','lag_1','lag_2','Diffavg','Inc1','Inc2','Incavg','Month']
dataset = CactusImageDataset('.. /input/aerial-cactus-identification/train.csv','.. /input/aerial-cactus-identification/train/train/', image_transforms['train'] )<split>
x_train = data[data['Date']<='2020-04-14']
y_train = x_train['Target']
x_val = data[(data['Date']>='2020-04-02')&(data['Date']<='2020-04-14')]
y_val = x_val['Target']
test_ = data[data['part']=='test']
train_size = int(( 0.80 * dataset.__len__()))
dev_size = dataset.__len__() - train_size
train_set, dev_set = random_split(dataset, [train_size, dev_size] )<load_pretrained>
from xgboost import DMatrix,train,plot_importance,XGBRegressor
BATCH_SIZE = 32
train_loader = DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True)
 dev_loader = DataLoader(dev_set, batch_size=BATCH_SIZE, shuffle=True )<choose_model_class>
params = {'objective': 'reg:squarederror',
'n_jobs': -1,
'seed': 236,
}
class CactusIdentifier(nn.Module):

 def __init__(self):
super(CactusIdentifier, self ).__init__()
self.conv1 = nn.Conv2d(3,16,3, padding = 1)
 self.conv2 = nn.Conv2d(16,32,3, padding = 1)
 self.conv3 = nn.Conv2d(32,64,3, padding = 1)
 self.conv4 = nn.Conv2d(64,128,3, padding = 1)
 self.pool = nn.MaxPool2d(2,2)
 self.fc1 = nn.Linear(128*2*2, 256)
 self.fc2 = nn.Linear(256, 64)
 self.fc3 = nn.Linear(64,2)
 self.act = nn.ReLU()
self.drop = nn.Dropout()
def forward(self, inp_image):
out = self.pool(self.act(self.conv1(inp_image)))
 out = self.pool(self.act(self.conv2(out)))
 out = self.pool(self.act(self.conv3(out)))
 out = self.pool(self.act(self.conv4(out)))
 out = out.view(-1, 128*2*2)
 out = self.drop(out)
 out = self.act(self.fc1(out))
out = self.drop(out)
 out = self.act(self.fc2(out))
out = self.drop(out)
 out = self.fc3(out)
 return out 
<choose_model_class>
from tqdm import tqdm
class TrainingModule() :

 def __init__(self, model):
self.model = model
self.loss_fn = nn.CrossEntropyLoss()
self.optimizer = optim.Adam(self.model.parameters())
 self.is_cuda = False
if torch.cuda.is_available() :
self.is_cuda = True
self.model = model.cuda()
def train_epoch(self, epoch, train_iterator):
total_loss = 0
stats_string = "Epoch : {:3d} | Iteration : {:4d} | Loss : {:4.4f}"
for i, data in enumerate(train_iterator):
inputs = data['image']
labels = data['label'].long()
if self.is_cuda:
inputs = inputs.cuda()
labels = labels.cuda()
self.optimizer.zero_grad()
outputs = self.model(inputs)
 loss = self.loss_fn(outputs, labels)
 loss.backward()
self.optimizer.step()
total_loss += loss.item()
if i % 100 == 0:
print(stats_string.format(epoch+1, i, total_loss/(i+1)))
 total_loss = 0
def train_model(self, train_iterator, dev_iterator, num_epocs = 30):
self.model.train()
min_loss = 1000000
stats_string = "Epoch : {:2d} | Train Loss : {:4.4f} | Dev Loss : {:4.4f}"
for i in range(num_epocs):
self.train_epoch(i, train_iterator)
 dev_loss = self.evaluate(dev_iterator)
 train_loss = self.evaluate(train_iterator)
 print(stats_string.format(i+1, train_loss, dev_loss))
if dev_loss <= min_loss:
torch.save(self.model.state_dict() , 'best_model')
 min_loss = dev_loss
print("Finished Training")
 def evaluate(self, iterator):
self.model.eval()
loss_total = 0
for i, data in enumerate(iterator):
inputs = data['image']
labels = data['label'].long()
if self.is_cuda:
inputs = inputs.cuda()
labels = labels.cuda()
preds = self.model(inputs)
 loss = self.loss_fn(preds, labels)
 loss_total += loss.item()
return loss_total 
<choose_model_class>
def feature(test_,ctr):
if ctr==1:
lags = [1]
elif ctr==2:
lags = [1,2]
elif ctr==3:
lags = [1,2,3]
else:
lags = [1,2,3,4]
for i in lags:
test_['lag_'+str(i)] = test_.groupby(['Location','variable'])['Target'].transform(lambda x: x.shift(i))
test_['Diff1'] = test_['lag_1'] - test_['lag_2']
test_['Diff2'] = test_['lag_2'] - test_['lag_3']
test_['Diff3'] = test_['lag_3'] - test_['lag_4']
test_['Diffavg'] =(test_['Diff1'] + test_['Diff2'] +test_['Diff3'])/3
test_["Inc1"] =(test_['Diff1'] / test_['lag_2'])*100
test_["Inc2"] =(test_['Diff2'] / test_['lag_3'])*100
test_["Inc3"] =(test_['Diff3'] / test_['lag_4'])*100
test_['Incavg'] =(test_['Inc1'] + test_['Inc2'] +test_['Inc3'])/3
test_.drop(['Diff1','Diff2','Diff3'],axis=1,inplace=True)
 ctr+=1
return test_
model = CactusIdentifier()
print(model )<train_model>
train_set = DMatrix(x_train[features],y_train)
 val_set = DMatrix(x_val[features],y_val)
 model = train(params,train_set,num_boost_round=100,evals=[(val_set,'validation')],verbose_eval=50)
 ctr = 1
for j in days:
test_set = DMatrix(test_[test_['Day']==j][features])
 test_.loc[test_['Day']==j,'Target'] = model.predict(test_set)
 test_ = feature(test_,ctr)
 ctr+=1
trainer = TrainingModule(model)
 trainer.train_model(train_loader, dev_loader, num_epocs = 1 )<load_from_csv>
sub = test_[['ForecastId','variable','Target']]
test_set = CactusImageDataset(".. /input/aerial-cactus-identification/sample_submission.csv", ".. /input/aerial-cactus-identification/test/test", image_transforms['test'])
 test_loader = DataLoader(test_set, batch_size=BATCH_SIZE, shuffle=False )<save_to_csv>
sub = pd.pivot(sub,index='ForecastId',columns='variable',values='Target' ).reset_index()
model.load_state_dict(torch.load('best_model'))
final_preds = []
for i, data in enumerate(test_loader):
inputs = data['image']
labels = data['label'].long()
if torch.cuda.is_available() :
inputs = inputs.cuda()
labels = labels.cuda()
preds = model(inputs)
 final_preds += preds[:,1].tolist()
test_set.cactus_annotations['has_cactus'] = final_preds
test_set.cactus_annotations.to_csv("samplesubmission.csv", index=False )<load_from_csv>
sub['ForecastId'] = sub['ForecastId'].astype(int)
 sub.columns = ['ForecastId','ConfirmedCases','Fatalities']
%%time
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/sample_submission.csv')
 df_train.head(3 )<define_variables>
sub.to_csv("submission.csv",index=False )