Kaggle competition: COVID19 Global Forecasting (Week 2)
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)<save_to_csv>
DT=1
optimize_model=False
Make_submission=True
n_estimators=200
max_depth=2
sub['toxic'] = model.predict(test_dataset)
 sub.head()
sub.to_csv('submission.csv', index=False )<import_modules>
train=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv")
 train.head()
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam,Adagrad
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors<categorify>
train[train['Province_State'].notna() ].groupby(['Country_Region'], sort=False)['Province_State'].nunique()
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):
tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )<categorify>
def add_location(df_old):
df=df_old.copy()
df['Date']=pd.to_datetime(df['Date'])
 df['Country_Region']=df['Country_Region'].fillna('')
 df['Province_State']=df['Province_State'].fillna('')
 df['location']=df['Province_State'].astype('str')+" "+df['Country_Region'].astype('str')
 return df
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )<choose_model_class>
train=add_location(train )
def build_model(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<set_options>
max_cases_old=train[train['Date']<'2020-03-22'].groupby(['location'], sort=False)['ConfirmedCases'].max()
max_cases=train.groupby(['location'], sort=False)['ConfirmedCases'].max()
print("Now: {}\r
Seven Days ago: {}".format(len(max_cases[max_cases<5]),len(max_cases_old[max_cases_old<5])) )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
train.set_index('location',inplace=True)
 train['day_of_year']=train['Date'].dt.dayofyear
train['day_of_week']=train['Date'].dt.dayofweek
first_day=train[(train['ConfirmedCases']>0)].groupby(['location'], sort=False)['day_of_year'].min()
first_day.rename('first_day',inplace=True )
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')
 EPOCHS = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-base'<load_pretrained>
def add_days_passed(df_old,first_day):
df=df_old.copy()
df=pd.concat([df,first_day],axis=1,join='inner')
 df['days_passed']=df['day_of_year']-df['first_day']
df.drop(columns=['first_day'],inplace=True)
 df['location']=df.index
df.set_index('Id',inplace=True)
 df['Id']=df.index
return df
tokenizer = AutoTokenizer.from_pretrained(MODEL )<load_from_csv>
train=add_days_passed(train,first_day)
 train.head()
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )<concatenate>
country_stat=pd.read_csv('.. /input/countryinfo/covid19countryinfo.csv')
 country_stat = country_stat[country_stat['region'].isnull() ]
def add_country_stat(old_df,country_stat):
df=old_df.copy()
df=df.merge(country_stat[['country','pop','medianage','sex65plus','lung','smokers','density']],left_on=['Country_Region'],right_on=['country'],how='left')
 df.drop(columns=['country'],inplace=True)
 df['pop']=df['pop'].fillna(1000)
 df['pop']=df['pop'].apply(lambda x: int(str(x ).replace(',', '')))
 df['density']=df['density'].fillna(0)
 df['medianage']=df['medianage'].fillna(0)
 df['sex65plus']=df['sex65plus'].fillna(1)
 df['lung']=df['lung'].fillna(24)
 df['smokers']=df['smokers'].fillna(24)
 return df
train=add_country_stat(train,country_stat )
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=100000, random_state=0)
 ] )<prepare_x_and_y>
border_info=pd.read_csv("https://raw.githubusercontent.com/geodatasource/country-borders/master/GEODATASOURCE-COUNTRY-BORDERS.CSV")
 border_info.drop(columns=["country_code","country_border_code"],inplace=True)
 border_info.replace({'United States of America':'US',
'United Kingdom of Great Britain and Northern Ireland':'United Kingdom',
'Bolivia(Plurinational State Of)':'Bolivia',
'Brunei Darussalam':'Brunei',
'Gambia(the)':'Gambia',
'Congo':'Congo(Kinshasa)',
'Cote d’Ivoire':"Cote d'Ivoire",
"Iran(Islamic Republic of)":'Iran',
"Korea(the Republic of)":'Korea, South',
"Lao People's Democratic Republic":'Laos',
"Moldova(the Republic of)":'Moldova',
"Russian Federation":'Russia',
"Syrian Arab Republic":'Syria',
"Taiwan(Province of China)":'Taiwan*',
"Tanzania(the United Republic of)":'Tanzania',
"Venezuela(Bolivarian Republic of)":'Venezuela',
"Viet Nam":'Vietnam'},inplace=True)
 border_info=border_info.fillna("")
 border_info.to_csv("border_info.csv")
%%time
x_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values<define_variables>
def expand_grid(data_dict):
rows = it_product(*data_dict.values())
 return pd.DataFrame.from_records(rows, columns=data_dict.keys() )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<load_pretrained>
country_info=train.groupby(['Date','Country_Region'])['ConfirmedCases'].sum()
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()<train_model>
skel=expand_grid({'Index':border_info.index,'Date':train['Date'].unique() })
 skel=skel.merge(border_info, how='inner', left_on=['Index'],right_index=True)
 skel=skel.merge(country_info, how='inner',
left_on=['Date','country_border_name'],right_on=['Date','Country_Region'] )
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
verbose = 1
)<train_model>
skel['Date']=skel['Date']+timedelta(days=DT)
 border_cases=skel.groupby(['country_name','Date'])['ConfirmedCases'].sum()
len(skel['country_name'].unique() )
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)<save_to_csv>
train=train.merge(border_cases, how='left', left_on=['Country_Region','Date'],right_on=['country_name','Date'])
 train['ConfirmedCases_y']=train['ConfirmedCases_y'].fillna(0)
 train.rename(columns={'ConfirmedCases_y':'ConfirmedCases_neighbors','ConfirmedCases_x':'ConfirmedCases'},inplace=True )
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )<install_modules>
big_train = pd.concat([train,pd.get_dummies(train['location'], prefix='loc')],axis=1)
 big_train['ConfirmedCases_neighbors']=np.log1p(big_train['ConfirmedCases_neighbors'])
 big_train.reset_index(inplace=True)
 big_train.drop(columns=["Id"],inplace=True )
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py > /dev/null
!python pytorch-xla-env-setup.py --version 20200420 --apt-packages libomp5 libopenblas-dev > /dev/null
!pip install transformers==2.5.1 > /dev/null
!pip install pandarallel > /dev/null
!pip install catalyst==20.4.2 > /dev/null<set_options>
def df_add_deltas(df_old):
df=df_old.copy()
df=df.sort_values(by=['location', 'Date'])
 df['d_ConfirmedCases'] = df.groupby(['location'])['ConfirmedCases'].diff()
df['d_Fatalities'] = df.groupby(['location'])['Fatalities'].diff()
df.loc[df['d_Fatalities']<0,'d_Fatalities']=0
df.loc[df['d_ConfirmedCases']<0,'d_ConfirmedCases']=0
df['prev_ConfirmedCases']=df['ConfirmedCases']-df['d_ConfirmedCases']
df['prev_Fatalities']=df['Fatalities']-df['d_Fatalities']
df['prev_ConfirmedCases']=np.log1p(df['prev_ConfirmedCases'])
 df['prev_Fatalities']=np.log1p(df['prev_Fatalities'])
 first_day_stat=df[df['Date']=='2020-01-22']
df.drop(df[df['Date']=='2020-01-22'].index, inplace=True)
 return df,first_day_stat
os.environ['XLA_USE_BF16'] = "1"
tqdm.pandas()
nltk.download('punkt')
 pandarallel.initialize(nb_workers=4, progress_bar=False )<define_variables>
big_train,first_day_stat=df_add_deltas(big_train )
SEED = 42
MAX_LENGTH = 224
BACKBONE_PATH = 'xlm-roberta-large'
ROOT_PATH = f'.. '
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )<string_transform>
big_train.reset_index(inplace=True,drop=True )
LANGS = {
'en': 'english',
'it': 'italian',
'fr': 'french',
'es': 'spanish',
'tr': 'turkish',
'ru': 'russian',
'pt': 'portuguese'
}
def get_sentences(text, lang='en'):
return sent_tokenize(text, LANGS.get(lang, 'english'))
def exclude_duplicate_sentences(text, lang='en'):
sentences = []
for sentence in get_sentences(text, lang):
sentence = sentence.strip()
if sentence not in sentences:
sentences.append(sentence)
 return ' '.join(sentences)
 def clean_text(text, lang='en'):
text = str(text)
 text = re.sub(r'[0-9"]', '', text)
 text = re.sub(r'
text = re.sub(r'@[\S]+\b', '', text)
 text = re.sub(r'https?\S+', '', text)
 text = re.sub(r'\s+', ' ', text)
 text = exclude_duplicate_sentences(text, lang)
 return text.strip()
class NLPTransform(BasicTransform):

 @property
def targets(self):
return {"data": self.apply}
def update_params(self, params, **kwargs):
if hasattr(self, "interpolation"):
params["interpolation"] = self.interpolation
if hasattr(self, "fill_value"):
params["fill_value"] = self.fill_value
return params
def get_sentences(self, text, lang='en'):
return sent_tokenize(text, LANGS.get(lang, 'english'))
class ShuffleSentencesTransform(NLPTransform):

 def __init__(self, always_apply=False, p=0.5):
super(ShuffleSentencesTransform, self ).__init__(always_apply, p)
 def apply(self, data, **params):
text, lang = data
sentences = self.get_sentences(text, lang)
 random.shuffle(sentences)
 return ' '.join(sentences), lang
class ExcludeDuplicateSentencesTransform(NLPTransform):

 def __init__(self, always_apply=False, p=0.5):
super(ExcludeDuplicateSentencesTransform, self ).__init__(always_apply, p)
 def apply(self, data, **params):
text, lang = data
sentences = []
for sentence in self.get_sentences(text, lang):
sentence = sentence.strip()
if sentence not in sentences:
sentences.append(sentence)
 return ' '.join(sentences), lang
class ExcludeNumbersTransform(NLPTransform):

 def __init__(self, always_apply=False, p=0.5):
super(ExcludeNumbersTransform, self ).__init__(always_apply, p)
 def apply(self, data, **params):
text, lang = data
text = re.sub(r'[0-9]', '', text)
 text = re.sub(r'\s+', ' ', text)
 return text, lang
class ExcludeHashtagsTransform(NLPTransform):
Exclude @users Exclude urls<load_from_csv>
X=big_train.drop(columns=['Province_State','Country_Region','Date','ConfirmedCases','Fatalities','location',
'd_ConfirmedCases','d_Fatalities'])
 y=big_train['d_ConfirmedCases']
y_2=big_train['d_Fatalities']
class SynthesicOpenSubtitlesTransform(NLPTransform):
def __init__(self, always_apply=False, p=0.5):
super(SynthesicOpenSubtitlesTransform, self ).__init__(always_apply, p)
 df = pd.read_csv(f'{ROOT_PATH}/input/open-subtitles-toxic-pseudo-labeling/open-subtitles-synthesic.csv', index_col='id')[['comment_text', 'toxic', 'lang']]
df = df[~df['comment_text'].isna() ]
df['comment_text'] = df.parallel_apply(lambda x: clean_text(x['comment_text'], x['lang']), axis=1)
 df = df.drop_duplicates(subset='comment_text')
 df['toxic'] = df['toxic'].round().astype(np.int)
 self.synthesic_toxic = df[df['toxic'] == 1].comment_text.values
self.synthesic_non_toxic = df[df['toxic'] == 0].comment_text.values
del df
gc.collect() ;
def generate_synthesic_sample(self, text, toxic):
texts = [text]
if toxic == 0:
for i in range(random.randint(1,5)) :
texts.append(random.choice(self.synthesic_non_toxic))
else:
for i in range(random.randint(0,2)) :
texts.append(random.choice(self.synthesic_non_toxic))
for i in range(random.randint(1,3)) :
texts.append(random.choice(self.synthesic_toxic))
random.shuffle(texts)
 return ' '.join(texts)
 def apply(self, data, **params):
text, toxic = data
text = self.generate_synthesic_sample(text, toxic)
 return text, toxic<feature_engineering>
max_day=X['day_of_year'].max()
mask_train=X['day_of_year']<max_day-DT+1
mask_test=X['day_of_year']>=max_day-DT+1
def get_train_transforms() :
return albumentations.Compose([
ExcludeUsersMentionedTransform(p=0.95),
ExcludeUrlsTransform(p=0.95),
ExcludeNumbersTransform(p=0.95),
ExcludeHashtagsTransform(p=0.95),
ExcludeDuplicateSentencesTransform(p=0.95),
], p=1.0)
 def get_synthesic_transforms() :
return SynthesicOpenSubtitlesTransform(p=0.5)
 train_transforms = get_train_transforms() ;
synthesic_transforms = get_synthesic_transforms()
tokenizer = XLMRobertaTokenizer.from_pretrained(BACKBONE_PATH)
 shuffle_transforms = ShuffleSentencesTransform(always_apply=True )<categorify>
X_train=X[mask_train]
X_test=X[mask_test]
y_train=y[mask_train]
y_test=y[mask_test]
y_train_2=y_2[mask_train]
y_test_2=y_2[mask_test]
def onehot(size, target):
vec = torch.zeros(size, dtype=torch.float32)
 vec[target] = 1.
 return vec
class DatasetRetriever(Dataset):
def __init__(self, labels_or_ids, comment_texts, langs, use_train_transforms=False, test=False):
self.test = test
self.labels_or_ids = labels_or_ids
self.comment_texts = comment_texts
self.langs = langs
self.use_train_transforms = use_train_transforms
def get_tokens(self, text):
encoded = tokenizer.encode_plus(
 text,
add_special_tokens=True,
max_length=MAX_LENGTH,
pad_to_max_length=True 
)
 return encoded['input_ids'], encoded['attention_mask']
def __len__(self):
return self.comment_texts.shape[0]
def __getitem__(self, idx):
text = self.comment_texts[idx]
lang = self.langs[idx]
if self.test is False:
label = self.labels_or_ids[idx]
target = onehot(2, label)
 if self.use_train_transforms:
text, _ = train_transforms(data=(text, lang)) ['data']
tokens, attention_mask = self.get_tokens(str(text))
token_length = sum(attention_mask)
 if token_length > 0.8*MAX_LENGTH:
text, _ = shuffle_transforms(data=(text, lang)) ['data']
elif token_length < 60:
text, _ = synthesic_transforms(data=(text, label)) ['data']
else:
tokens, attention_mask = torch.tensor(tokens), torch.tensor(attention_mask)
 return target, tokens, attention_mask
tokens, attention_mask = self.get_tokens(str(text))
tokens, attention_mask = torch.tensor(tokens), torch.tensor(attention_mask)
 if self.test is False:
return target, tokens, attention_mask
return self.labels_or_ids[idx], tokens, attention_mask
def get_labels(self):
return list(np.char.add(self.labels_or_ids.astype(str), self.langs))<load_from_csv>
X_test['day_of_year'].nunique()
%%time
df_train = pd.read_csv(f'{ROOT_PATH}/input/jigsaw-public-baseline-train-data/train_data.csv')
 train_dataset = DatasetRetriever(
 labels_or_ids=df_train['toxic'].values,
comment_texts=df_train['comment_text'].values,
langs=df_train['lang'].values,
use_train_transforms=True, 
)
 del df_train
gc.collect() ;
for targets, tokens, attention_masks in train_dataset:
break
print(targets)
 print(tokens.shape)
 print(attention_masks.shape )<load_from_csv>
X_train.drop(columns=['day_of_year'],inplace=True)
 X_test.drop(columns=['day_of_year'],inplace=True)
 X_train.drop(columns=['day_of_week'],inplace=True)
 X_test.drop(columns=['day_of_week'],inplace=True)
 X.drop(columns=['day_of_year'],inplace=True)
 X.drop(columns=['day_of_week'],inplace=True)
df_val = pd.read_csv(f'{ROOT_PATH}/input/jigsaw-multilingual-toxic-comment-classification/validation.csv', index_col='id')
 validation_tune_dataset = DatasetRetriever(
 labels_or_ids=df_val['toxic'].values,
comment_texts=df_val['comment_text'].values,
langs=df_val['lang'].values,
use_train_transforms=True, 
)
 df_val['comment_text'] = df_val.parallel_apply(lambda x: clean_text(x['comment_text'], x['lang']), axis=1)
 validation_dataset = DatasetRetriever(
 labels_or_ids=df_val['toxic'].values,
comment_texts=df_val['comment_text'].values,
langs=df_val['lang'].values,
use_train_transforms=False, 
)
 del df_val
gc.collect() ;
for targets, tokens, attention_masks in validation_dataset:
break
print(targets)
 print(tokens.shape)
 print(attention_masks.shape )<load_from_csv>
X.drop(columns=['index'],inplace=True)
 X_train.drop(columns=['index'],inplace=True)
 X_test.drop(columns=['index'],inplace=True )
df_test = pd.read_csv(f'{ROOT_PATH}/input/jigsaw-multilingual-toxic-comment-classification/test.csv', index_col='id')
 df_test['comment_text'] = df_test.parallel_apply(lambda x: clean_text(x['content'], x['lang']), axis=1)
 test_dataset = DatasetRetriever(
 labels_or_ids=df_test.index.values,
comment_texts=df_test['comment_text'].values,
langs=df_test['lang'].values,
use_train_transforms=False,
test=True 
)
 del df_test
gc.collect() ;
for ids, tokens, attention_masks in test_dataset:
break
print(ids)
 print(tokens.shape)
 print(attention_masks.shape )<compute_train_metric>
if optimize_model:
model = xgb.XGBRegressor(random_state=42)
 n_estimators_grid = [500, 1000,1500]
max_depth_grid = [6, 8, 10]
param_grid = dict(max_depth=max_depth_grid, n_estimators=n_estimators_grid)
 grid_search = GridSearchCV(model, param_grid, scoring="neg_root_mean_squared_error", n_jobs=-1, cv=[(X[mask_train].index,X[mask_test].index)], verbose=1)
 grid_result = grid_search.fit(X,np.log1p(y))
print("Best: %f using %s" %(grid_result.best_score_, grid_result.best_params_))
print(grid_result.cv_results_ )
class RocAucMeter(object):
def __init__(self):
self.reset()
def reset(self):
self.y_true = np.array([0,1])
 self.y_pred = np.array([0.5,0.5])
 self.score = 0
def update(self, y_true, y_pred):
y_true = y_true.cpu().numpy().argmax(axis=1)
 y_pred = nn.functional.softmax(y_pred, dim=1 ).data.cpu().numpy() [:,1]
self.y_true = np.hstack(( self.y_true, y_true))
self.y_pred = np.hstack(( self.y_pred, y_pred))
self.score = sklearn.metrics.roc_auc_score(self.y_true, self.y_pred, labels=np.array([0, 1]))
@property
def avg(self):
return self.score
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count<normalization>
if optimize_model:
model = xgb.XGBRegressor(random_state=42)
 n_estimators_grid = [400,500,600]
max_depth_grid = [5,6,7]
param_grid = dict(max_depth=max_depth_grid, n_estimators=n_estimators_grid)
 grid_search = GridSearchCV(model, param_grid, scoring="neg_root_mean_squared_error", n_jobs=-1, cv=[(X[mask_train].index,X[mask_test].index)], verbose=1)
 grid_result = grid_search.fit(X,np.log1p(y))
print("Best: %f using %s" %(grid_result.best_score_, grid_result.best_params_))
print(grid_result.cv_results_ )
class LabelSmoothing(nn.Module):
def __init__(self, smoothing = 0.1):
super(LabelSmoothing, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
def forward(self, x, target):
if self.training:
x = x.float()
target = target.float()
logprobs = torch.nn.functional.log_softmax(x, dim = -1)
 nll_loss = -logprobs * target
nll_loss = nll_loss.sum(-1)
 smooth_loss = -logprobs.mean(dim=-1)
 loss = self.confidence * nll_loss + self.smoothing * smooth_loss
return loss.mean()
else:
return torch.nn.functional.cross_entropy(x, target )<set_options>
if optimize_model:
model = xgb.XGBRegressor(random_state=42)
 n_estimators_grid = [150,200,250]
max_depth_grid = [1,2,3]
param_grid = dict(max_depth=max_depth_grid, n_estimators=n_estimators_grid)
 grid_search = GridSearchCV(model, param_grid, scoring="neg_root_mean_squared_error", n_jobs=-1, cv=[(X[mask_train].index,X[mask_test].index)], verbose=1)
 grid_result = grid_search.fit(X,np.log1p(y))
print("Best: %f using %s" %(grid_result.best_score_, grid_result.best_params_))
print(grid_result.cv_results_ )
warnings.filterwarnings("ignore")
 class TPUFitter:
def __init__(self, model, device, config):
if not os.path.exists('node_submissions'):
os.makedirs('node_submissions')
 self.config = config
self.epoch = 0
self.log_path = 'log.txt'
self.model = model
self.device = device
param_optimizer = list(self.model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
self.optimizer = AdamW(optimizer_grouped_parameters, lr=config.lr*xm.xrt_world_size())
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 self.criterion = config.criterion
xm.master_print(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
para_loader = pl.ParallelLoader(train_loader, [self.device])
 losses, final_scores = self.train_one_epoch(para_loader.per_device_loader(self.device))
self.log(f'[RESULT]: Train.Epoch: {self.epoch}, loss: {losses.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')
 t = time.time()
para_loader = pl.ParallelLoader(validation_loader, [self.device])
 losses, final_scores = self.validation(para_loader.per_device_loader(self.device))
self.log(f'[RESULT]: Validation.Epoch: {self.epoch}, loss: {losses.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')
 if self.config.validation_scheduler:
self.scheduler.step(metrics=final_scores.avg)
 self.epoch += 1
def run_tuning_and_inference(self, test_loader, validation_tune_loader):
for e in range(2):
self.optimizer.param_groups[0]['lr'] = self.config.lr*xm.xrt_world_size() /(e + 1)
 para_loader = pl.ParallelLoader(validation_tune_loader, [self.device])
 losses, final_scores = self.train_one_epoch(para_loader.per_device_loader(self.device))
para_loader = pl.ParallelLoader(test_loader, [self.device])
 self.run_inference(para_loader.per_device_loader(self.device))
def validation(self, val_loader):
self.model.eval()
losses = AverageMeter()
final_scores = RocAucMeter()
t = time.time()
for step,(targets, inputs, attention_masks)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
xm.master_print(
 f'Valid Step {step}, loss: ' + \
f'{losses.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}' 
)
 with torch.no_grad() :
inputs = inputs.to(self.device, dtype=torch.long)
 attention_masks = attention_masks.to(self.device, dtype=torch.long)
 targets = targets.to(self.device, dtype=torch.float)
 outputs = self.model(inputs, attention_masks)
 loss = self.criterion(outputs, targets)
 batch_size = inputs.size(0)
 final_scores.update(targets, outputs)
 losses.update(loss.detach().item() , batch_size)
 return losses, final_scores
def train_one_epoch(self, train_loader):
self.model.train()
losses = AverageMeter()
final_scores = RocAucMeter()
t = time.time()
for step,(targets, inputs, attention_masks)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
self.log(
 f'Train Step {step}, loss: ' + \
f'{losses.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}' 
)
 inputs = inputs.to(self.device, dtype=torch.long)
 attention_masks = attention_masks.to(self.device, dtype=torch.long)
 targets = targets.to(self.device, dtype=torch.float)
 self.optimizer.zero_grad()
outputs = self.model(inputs, attention_masks)
 loss = self.criterion(outputs, targets)
 batch_size = inputs.size(0)
 final_scores.update(targets, outputs)
 losses.update(loss.detach().item() , batch_size)
 loss.backward()
xm.optimizer_step(self.optimizer)
 if self.config.step_scheduler:
self.scheduler.step()
self.model.eval()
self.save('last-checkpoint.bin')
 return losses, final_scores
def run_inference(self, test_loader):
self.model.eval()
result = {'id': [], 'toxic': []}
t = time.time()
for step,(ids, inputs, attention_masks)in enumerate(test_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
xm.master_print(f'Prediction Step {step}, time: {(time.time() - t):.5f}')
 with torch.no_grad() :
inputs = inputs.to(self.device, dtype=torch.long)
 attention_masks = attention_masks.to(self.device, dtype=torch.long)
 outputs = self.model(inputs, attention_masks)
 toxics = nn.functional.softmax(outputs, dim=1 ).data.cpu().numpy() [:,1]
result['id'].extend(ids.cpu().numpy())
 result['toxic'].extend(toxics)
 result = pd.DataFrame(result)
 node_count = len(glob('node_submissions/*.csv'))
result.to_csv(f'node_submissions/submission_{node_count}_{datetime.utcnow().microsecond}_{random.random() }.csv', index=False)
 def save(self, path):
xm.save(self.model.state_dict() , path)
 def log(self, message):
if self.config.verbose:
xm.master_print(message)
 with open(self.log_path, 'a+')as logger:
xm.master_print(f'{message}', logger )<categorify>
reg = xgb.XGBRegressor(n_estimators=n_estimators,max_depth=max_depth,random_state=42)
 reg_2 = xgb.XGBRegressor(n_estimators=n_estimators,max_depth=max_depth,random_state=42 )
class ToxicSimpleNNModel(nn.Module):
def __init__(self):
super(ToxicSimpleNNModel, self ).__init__()
self.backbone = XLMRobertaModel.from_pretrained(BACKBONE_PATH)
 self.dropout = nn.Dropout(0.3)
 self.linear = nn.Linear(
 in_features=self.backbone.pooler.dense.out_features*2,
out_features=2, 
)
 def forward(self, input_ids, attention_masks):
bs, seq_length = input_ids.shape
seq_x, _ = self.backbone(input_ids=input_ids, attention_mask=attention_masks)
 apool = torch.mean(seq_x, 1)
 mpool, _ = torch.max(seq_x, 1)
 x = torch.cat(( apool, mpool), 1)
 x = self.dropout(x)
 return self.linear(x )<choose_model_class>
reg.fit(X_train,np.log1p(y_train))
net = ToxicSimpleNNModel()<init_hyperparams>
y_pred = reg.predict(X_test )
class TrainGlobalConfig:
num_workers = 0
batch_size = 16
n_epochs = 3
lr = 0.5 * 1e-5
verbose = True
verbose_step = 50
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(
 mode='max',
factor=0.7,
patience=0,
verbose=False,
threshold=0.0001,
threshold_mode='abs',
cooldown=0,
min_lr=1e-8,
eps=1e-08 
)
 criterion = LabelSmoothing() 
<define_search_model>
np.sqrt(mean_squared_error(y_pred,np.log1p(y_test)) )
def _mp_fn(rank, flags):
device = xm.xla_device()
net.to(device)
 train_sampler = DistributedSamplerWrapper(
 sampler=BalanceClassSampler(labels=train_dataset.get_labels() , mode="downsampling"),
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=True 
)
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=train_sampler,
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers, 
)
 validation_sampler = torch.utils.data.distributed.DistributedSampler(
 validation_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False 
)
 validation_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=validation_sampler,
pin_memory=False,
drop_last=False,
num_workers=TrainGlobalConfig.num_workers 
)
 validation_tune_sampler = torch.utils.data.distributed.DistributedSampler(
 validation_tune_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=True 
)
 validation_tune_loader = torch.utils.data.DataLoader(
 validation_tune_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=validation_tune_sampler,
pin_memory=False,
drop_last=False,
num_workers=TrainGlobalConfig.num_workers 
)
 test_sampler = torch.utils.data.distributed.DistributedSampler(
 test_dataset,
num_replicas=xm.xrt_world_size() ,
rank=xm.get_ordinal() ,
shuffle=False 
)
 test_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=test_sampler,
pin_memory=False,
drop_last=False,
num_workers=TrainGlobalConfig.num_workers 
)
 if rank == 0:
time.sleep(1)
 fitter = TPUFitter(model=net, device=device, config=TrainGlobalConfig)
 fitter.fit(train_loader, validation_loader)
 fitter.run_tuning_and_inference(test_loader, validation_tune_loader )<load_from_csv>
X_train_2=X_train.copy()
X_train_2['d_confirmed']=y_train
X_test_2=X_test.copy()
X_test_2['d_confirmed']=y_pred
file = open('.. /input/jigsaw-public-baseline-results/log.txt', 'r')
 for line in file.readlines() :
print(line[:-1])
 file.close()<load_from_csv>
reg_2.fit(X_train_2,np.log1p(y_train_2))
submission = pd.read_csv('.. /input/jigsaw-public-baseline-results/submission.csv', index_col='id')
 submission.hist(bins=100 )<save_to_csv>
y_pred_2 = reg_2.predict(X_test_2 )
submission.to_csv('submission.csv' )<import_modules>
np.sqrt(mean_squared_error(y_pred_2,np.log1p(y_test_2)) )
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors<categorify>
test=pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv")
 test.rename(columns={'ForecastId':'Id'},inplace=True)
 test=add_location(test)
 test.set_index('location',inplace=True)
 test['day_of_year']=test['Date'].dt.dayofyear
test['day_of_week']=test['Date'].dt.dayofweek
test=add_days_passed(test,first_day)
 test=add_country_stat(test,country_stat )
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):

 tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )<categorify>
big_train=big_train.drop(columns=["index"])
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )<choose_model_class>
if Make_submission==True:
results=[]
for d in days_to_predict:
print("Predicting {}".format(d))
if d in known:
print("Data Known")
 X=big_train.drop(columns=['Province_State','Country_Region','ConfirmedCases','Fatalities','location','Date',
'day_of_year','day_of_week','d_ConfirmedCases','d_Fatalities'])
 y=big_train['d_ConfirmedCases']
y_2=big_train['d_Fatalities']
mask_train=big_train['Date']<d
mask_val=big_train['Date']==d
X_train=X[mask_train]
y_train=y[mask_train]
y_train_2=y_2[mask_train]
X_val=X[mask_val]
y_val=y[mask_val]
y_val_2=y_2[mask_val]
reg = xgb.XGBRegressor(n_estimators=n_estimators,max_depth=max_depth,random_state=42)
 reg_2 = xgb.XGBRegressor(n_estimators=n_estimators,max_depth=max_depth,random_state=42)
 reg.fit(X_train,np.log1p(y_train))
y_pred = reg.predict(X_val)
 print("MSLE {}".format(mean_squared_error(y_pred,np.log1p(y_val))))
X_train_2=X_train.copy()
X_train_2['d_ConfirmedCases']=np.log1p(y_train)
 X_val_2=X_val.copy()
X_val_2['d_ConfirmedCases']=y_pred
reg_2.fit(X_train_2,np.log1p(y_train_2))
y_pred_2 = reg_2.predict(X_val_2)
 print("MSLE {}".format(mean_squared_error(y_pred_2,np.log1p(y_val_2))))
elif d-np.timedelta64(86400000000000,'ns')in known:
print("Data Known")
 X=big_train.drop(columns=['Province_State','Country_Region','ConfirmedCases','Fatalities','location','Date',
'day_of_year','day_of_week','d_ConfirmedCases','d_Fatalities'])
 y=big_train['d_ConfirmedCases']
y_2=big_train['d_Fatalities']
mask_train=big_train['Date']<d
X_train=X[mask_train]
y_train=y[mask_train]
y_train_2=y_2[mask_train]
reg = xgb.XGBRegressor(n_estimators=n_estimators,max_depth=max_depth,random_state=42)
 reg_2 = xgb.XGBRegressor(n_estimators=n_estimators,max_depth=max_depth,random_state=42)
 reg.fit(X_train,np.log1p(y_train))
X_train_2=X_train.copy()
X_train_2['d_ConfirmedCases']=np.log1p(y_train)
 reg_2.fit(X_train_2,np.log1p(y_train_2))
X_test=test[test['Date']==d]
day=X_test['day_of_year'].iloc[0]
country_info=big_train[big_train['day_of_year']==day-1].groupby(['Country_Region'])['ConfirmedCases'].sum()
border_cases=border_info.merge(country_info, how='inner',
left_on=['country_border_name'],right_on=['Country_Region'])
 border_cases=border_cases.groupby(['country_name'])['ConfirmedCases'].sum()
border_cases=border_cases.rename('ConfirmedCases_neighbors')
 X_test=X_test.merge(border_cases, how='left', left_on=['Country_Region'],right_on=['country_name'])
 X_test['ConfirmedCases_neighbors']=X_test['ConfirmedCases_neighbors'].fillna(0)
 X_test = pd.concat([X_test,pd.get_dummies(X_test['location'], prefix='loc')],axis=1)
 X_test['ConfirmedCases_neighbors']=np.log1p(X_test['ConfirmedCases_neighbors'])
 X_test=X_test.merge(big_train[big_train['day_of_year']==day-1][['location','ConfirmedCases','Fatalities']], how='left',
left_on=['location'],right_on=['location'])
 X_test.rename(columns={'ConfirmedCases':'prev_ConfirmedCases','Fatalities':'prev_Fatalities'},inplace=True)
 X_test['prev_ConfirmedCases']=np.log1p(X_test['prev_ConfirmedCases'])
 X_test['prev_Fatalities']=np.log1p(X_test['prev_Fatalities'])
 X_test.set_index('Id',inplace=True)
 y_test=reg.predict(X_test.drop(columns=['Province_State','Country_Region','location','Date','day_of_year','day_of_week']))
X_test['d_ConfirmedCases']=y_test
y_test=reg_2.predict(X_test.drop(columns=['Province_State','Country_Region','location','Date',
'day_of_year','day_of_week']))
X_test['d_Fatalities']=y_test
X_test['Id']=X_test.index
X_test=X_test.merge(big_train[big_train['day_of_year']==day-1][['location','ConfirmedCases','Fatalities']], how='left',
left_on=['location'],right_on=['location'])
 X_test.set_index('Id',inplace=True)
 print(X_test.head(5))
X_test['d_ConfirmedCases']=np.expm1(X_test['d_ConfirmedCases'])
 X_test['d_Fatalities']=np.expm1(X_test['d_Fatalities'])
 X_test['ConfirmedCases']+=X_test['d_ConfirmedCases']
X_test['Fatalities']+=X_test['d_Fatalities']
results.append(X_test[['ConfirmedCases','Fatalities']])
 if not d in known:
big_train=pd.concat([big_train,X_test])
<define_variables><EOS> 
if Make_submission==True:
submission=pd.read_csv(".. /input/covid19-global-forecasting-week-2/submission.csv")
 submission.drop(columns=['ConfirmedCases','Fatalities'],inplace=True)
 submission=submission.merge(pd.concat(results),left_on=['ForecastId'],right_index=True ).clip(lower=0)
 submission.to_csv('submission.csv',index=False )