Kaggle competition: COVID19 Global Forecasting (Week 2)
def get_bag_utility_distributions(candidate_bags):
distributions = []
size = len(candidate_bags)
 for i, candidate_bag in enumerate(candidate_bags):
if i % 7000 == 0:
sys.stdout.write("{:.4f}\r".format(float(i)/ float(size)))
 distributions.append(get_bag_utility_distribution(candidate_bag))
print("")
 return np.vstack(distributions)
 def get_bag_utility_distribution(candidate_bag):
bag = { gifts[i]: int(candidate_bag[i])for i in range(len(gifts)) if candidate_bag[i] > 0 }
sampler, name = create_bag_utility_sampler(bag)
 sample = sampler(10000)
 return np.mean(sample), np.var(sample)
 bag_utility_distributions = get_bag_utility_distributions(candidate_bags)
 print(bag_utility_distributions.shape )<define_variables>
import plotly.graph_objects as go
import matplotlib.pyplot as plt
from tqdm import tqdm
import time
from datetime import datetime
from pathlib import Path
from sklearn import preprocessing
import keras.backend as K
from keras.models import Sequential
from keras.layers import Dense, LSTM, Dropout, GRU
from keras.callbacks import EarlyStopping
from sklearn.preprocessing import StandardScaler, MinMaxScaler
num_gifts_available = {
"horse": 1000,
"ball": 1100,
"bike": 500,
"train": 1000,
"book": 1200,
"doll": 1000,
"blocks": 1000,
"gloves": 200,
"coal": 166
}<concatenate>
train = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/train.csv")
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-2/test.csv")
 train.tail()
def pack_linprog(bags, distributions, min_variance, max_bags=1000):
c = - distributions[:,0]
A_ub = bags.T
b_ub = np.array([num_gifts_available[gift] for gift in gifts])
 A_ub = np.vstack([A_ub, np.ones(A_ub.shape[1])])
 b_ub = np.hstack([b_ub, [max_bags]])
 if min_variance is not None:
A_ub = np.vstack([A_ub, -distributions[:,1]])
 b_ub = np.hstack([b_ub, [-min_variance]])
 result = linprog(c, A_ub=A_ub, b_ub=b_ub)
 if result["success"] == False:
return [], True
else:
return result["x"].astype('int64'), False
def pack_bags(bags, distributions, min_variance=None):
max_bags = 1000
bag_quantities, infeasible = pack_linprog(bags, distributions, min_variance=min_variance)
 while np.sum(bag_quantities)< 1000:
max_bags += 1
bag_quantities, infeasible = pack_linprog(bags, distributions, min_variance=min_variance, max_bags=max_bags)
 if max_bags > 1015:
print("WARNING: not getting 1000 bags")
 break
if infeasible:
continue
if infeasible:
print("infeasible")
 return [], [], []
chosen_bag_idx = np.where(bag_quantities)[0]
chosen_bags = bags[chosen_bag_idx]
chosen_distributions = distributions[chosen_bag_idx]
chosen_quantities = bag_quantities[chosen_bag_idx]
while np.sum(chosen_quantities)> 1000:
idx = np.random.randint(len(chosen_quantities))
chosen_quantities[idx] = max(chosen_quantities[idx]-1, 0)
 score_distribution = np.dot(chosen_quantities, chosen_distributions)
 print("{} bags - score distribution: mean = {:.2f} | var = {:.2f}" 
.format(np.sum(chosen_quantities), score_distribution[0], score_distribution[1]))
return chosen_bags, chosen_distributions, chosen_quantities
packed_bags, packed_distributions, packed_quantities \
= pack_bags(candidate_bags, bag_utility_distributions, min_variance=None )<feature_engineering>
train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train['Country_Region'] = train['Country_Region'].astype(str)
 test['Country_Region'] = test['Country_Region'].astype(str)
def evaluate_variances() :
results = {}
for i, min_variance in enumerate(np.linspace(100000, 410000, num=10)) :
bags, distributions, quantities = pack_bags(candidate_bags, bag_utility_distributions, min_variance=min_variance)
 results[min_variance] = np.dot(quantities, distributions)
 return results
scores_for_min_variance = evaluate_variances()<save_to_csv>
EMPTY_VAL = "EMPTY_VAL"
def fillState(state, country):
if state == EMPTY_VAL: return country
return state
train['Province_State'].fillna(EMPTY_VAL, inplace=True)
 train['Province_State'] = train.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : fillState(x['Province_State'], x['Country_Region']), axis=1)
 test['Province_State'].fillna(EMPTY_VAL, inplace=True)
 test['Province_State'] = test.loc[:, ['Province_State', 'Country_Region']].apply(lambda x : fillState(x['Province_State'], x['Country_Region']), axis=1 )
def create_submissions(bags, quantities, num_submissions=60):
def create_stock(n):
stock = { gift: list(map(lambda id: "{}_{}".format(gift, id),np.arange(num_gifts_available[gift])))for gift in gifts }
return shuffle(stock, n)
 def shuffle(stock, seed):
np.random.seed(seed)
 for gift in stock.keys() :
np.random.shuffle(stock[gift])
 return stock
def generate_submission(n):
stock = create_stock(n)
 with open("submission_{}.csv".format(n), 'w+')as submission_file:
submission_file.write('Gifts
')
 for i in range(len(bags)) :
for quantity in range(quantities[i]):
current_gifts = bags[i]
for gift_idx, gift_quantity in enumerate(current_gifts[:len(gifts)]):
gift_name = gifts[gift_idx]
for j in range(int(gift_quantity)) :
submission_file.write("{} ".format(stock[gift_name].pop()))
submission_file.write("
")
 for n in range(num_submissions):
generate_submission(n)
 create_submissions(packed_bags, packed_quantities )<define_variables>
le = preprocessing.LabelEncoder()
train['country_encoder'] = le.fit_transform(train['Country_Region'])
 train['date_int'] = train['Date'].apply(lambda x: datetime.strftime(x, '%m%d')).astype(int)
 test['country_encoder'] = le.transform(test['Country_Region'])
 test['date_int'] = test['Date'].apply(lambda x: datetime.strftime(x, '%m%d')).astype(int )
path="/kaggle/input/jigsaw-multilingual-toxic-comment-classification/"<load_from_csv>
le = preprocessing.LabelEncoder()
train['province_encoder'] = le.fit_transform(train['Province_State'])
 test['province_encoder'] = le.transform(test['Province_State'] )
training_set=pd.read_csv(path+"jigsaw-toxic-comment-train.csv")
 test_set=pd.read_csv(path+"test.csv")
 validation_set=pd.read_csv(path+"validation.csv")
 sub=pd.read_csv(path+"sample_submission.csv" )<count_missing_values>
start_time = time.time()
country = train['Country_Region'].drop_duplicates()
train_df = train.copy()
train_df.rename(columns={'Date': 'date', 'ConfirmedCases': 'cc_cases', 'Fatalities': 'ft_cases', 'Country_Region': 'country', 'Province_State': 'province'}, inplace=True)
 lags = np.arange(1,8,1)
 with parallel_backend('threading', n_jobs = -1):
with tqdm(total = len(list(train_df['date'].unique())))as pbar:
for d in train_df['date'].drop_duplicates() :
for i in country:
province = train_df[train_df['country'] == i]['province'].drop_duplicates()
for j in province:
mask =(train_df['date'] == d)&(train_df['country'] == i)&(train_df['province'] == j)
 for lag in lags:
mask_org =(train_df['date'] ==(d - pd.Timedelta(days=lag)))&(train_df['country'] == i)&(train_df['province'] == j)
 try:
train_df.loc[mask, 'cc_cases_' + str(lag)] = train_df.loc[mask_org, 'cc_cases'].values
except:
train_df.loc[mask, 'cc_cases_' + str(lag)] = 0
try:
train_df.loc[mask, 'ft_cases_' + str(lag)] = train_df.loc[mask_org, 'ft_cases'].values
except:
train_df.loc[mask, 'ft_cases_' + str(lag)] = 0
pbar.update(1)
 print('Time spent for building features is {} minutes'.format(round(( time.time() -start_time)/60,1)) )
training_set.isnull().sum()<count_missing_values>
train_df = pd.read_csv(Path('/kaggle/working/', 'train_df.csv'), index_col = 0, parse_dates = ['date'])
 train_df[train_df['country'] == 'Italy'].tail()
test_set.isnull().sum()<count_missing_values>
def split_train_val(df, val_ratio):
val_len = int(len(df)* val_ratio)
 train_set = df[:-val_len]
val_set = df[-val_len:]
return train_set, val_set
validation_set.isnull().sum()<drop_column>
test_fixed_cols = ['ForecastId', 'Province_State', 'Country_Region', 'Date']
fixed_cols = ['Id', 'province', 'country', 'date']
output_cols = ['cc_cases', 'ft_cases']
input_cols = list(set(train_df.columns.to_list())- set(fixed_cols)- set(output_cols))
print('output columns are ', output_cols)
 print('input columns are ', input_cols)
 X = train_df[input_cols]
y = train_df[output_cols]
training_set.drop(["severe_toxic","obscene","threat","insult","identity_hate","id"],axis=1,inplace=True)
 training_set.head()<drop_column>
cc_input = ['cc_cases_1', 'cc_cases_2', 'cc_cases_3', 'cc_cases_4', 'cc_cases_5', 'cc_cases_6', 'cc_cases_7', 'country_encoder', 'province_encoder', 'date_int']
ft_input = ['ft_cases_1', 'ft_cases_2', 'ft_cases_3', 'ft_cases_4', 'ft_cases_5', 'ft_cases_6', 'ft_cases_7', 'country_encoder', 'province_encoder', 'date_int']
cc_output = ['cc_cases']
ft_output = ['ft_cases']
X_cc = X[cc_input]
X_ft = X[ft_input]
y_cc = y[cc_output]
y_ft = y[ft_output]
train_X_cc, val_X_cc = split_train_val(df = X_cc, val_ratio = 0.1)
 train_y_cc, val_y_cc = split_train_val(df = y_cc, val_ratio = 0.1)
 train_X_ft, val_X_ft = split_train_val(df = X_ft, val_ratio = 0.1)
 train_y_ft, val_y_ft = split_train_val(df = y_ft, val_ratio = 0.1 )
test_set.drop(["lang","id"],axis=1,inplace=True)
 validation_set.drop(["lang","id"],axis=1,inplace=True )<categorify>
idx = np.random.RandomState(seed=42 ).permutation(train_X_cc.index)
 train_X_cc = train_X_cc.reindex(idx)
 train_y_cc = train_y_cc.reindex(idx)
 train_X_ft = train_X_ft.reindex(idx)
 train_y_ft = train_y_ft.reindex(idx)
def clean_text(text):
text=text.lower()
text=re.sub("
"," ",text)
 text=re.sub("\[\[User.*"," ",text)
 text=re.sub("\(http://.*?\s\(http://.*\)"," ",text)
 text=re.sub("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}"," ",text)
 text=re.sub("[.,]"," ",text)
 text=text.split()
text=" ".join(text)
 return text<feature_engineering>
X_scaler_cc = MinMaxScaler()
X_train_cc = X_scaler_cc.fit_transform(train_X_cc)
 X_val_cc = X_scaler_cc.transform(val_X_cc)
 y_scaler_cc = MinMaxScaler()
y_train_cc = y_scaler_cc.fit_transform(train_y_cc)
 y_val_cc = y_scaler_cc.transform(val_y_cc )
training_set["comment_text"]=training_set["comment_text"].apply(str ).apply(lambda x:clean_text(x))
test_set["content"]=test_set["content"].apply(str ).apply(lambda x:clean_text(x))
validation_set["comment_text"]=validation_set["comment_text"].apply(str ).apply(lambda x:clean_text(x))<import_modules>
X_scaler_ft = MinMaxScaler()
X_train_ft = X_scaler_ft.fit_transform(train_X_ft)
 X_val_ft = X_scaler_ft.transform(val_X_ft)
 y_scaler_ft = MinMaxScaler()
y_train_ft = y_scaler_ft.fit_transform(train_y_ft)
 y_val_ft = y_scaler_ft.transform(val_y_ft )
from keras.models import Sequential
from keras.layers import Embedding
from keras.layers import Dense
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Flatten
from keras.layers import SimpleRNN
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors 
<categorify>
X_train_cc = train_X_cc.to_numpy()
X_val_cc = val_X_cc.to_numpy()
X_train_ft = train_X_ft.to_numpy()
X_val_ft = val_X_ft.to_numpy()
y_train_cc = train_y_cc.to_numpy()
y_val_cc = val_y_cc.to_numpy()
y_train_ft = train_y_ft.to_numpy()
y_val_ft = val_y_ft.to_numpy()
def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):
tokenizer.enable_truncation(max_length=maxlen)
 tokenizer.enable_padding(max_length=maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )<categorify>
def root_mean_squared_log_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(K.log(y_pred + 1)- K.log(y_true + 1))))
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )<choose_model_class>
def GRU_model(n_1, input_dim, output_dim):
model = Sequential()
model.add(GRU(n_1,input_shape=(1, input_dim), activation='relu'))
model.add(Dropout(0.1))
model.add(Dense(output_dim, activation='relu'))
model.compile(loss=root_mean_squared_log_error, optimizer='adam')
 print(model.summary())
 return model
def build_model(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<define_variables>
model_cc = GRU_model(4, X_train_cc.shape[-1], y_train_cc.shape[-1])
 model_ft = GRU_model(4, X_train_ft.shape[-1], y_train_ft.shape[-1])
 early_stop = EarlyStopping(monitor='loss', patience=5, verbose=0, mode='min' )
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 3
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 256
MODEL = 'jplu/tf-xlm-roberta-large'<load_pretrained>
print('Start model training')
 start_time = time.time()
history_cc = model_cc.fit(X_train_cc, y_train_cc, epochs = 100,validation_data =(X_val_cc, y_val_cc), verbose = 2, callbacks=[early_stop])
 model_cc.save("model_cc.h5")
 print('Time spent for model training is {} minutes'.format(round(( time.time() -start_time)/60,1)) )
tokenizer = AutoTokenizer.from_pretrained(MODEL )<prepare_x_and_y>
print('Start model training')
 start_time = time.time()
history_ft = model_ft.fit(X_train_ft, y_train_ft, epochs = 100,validation_data =(X_val_ft, y_val_ft), verbose = 2, callbacks=[early_stop])
 model_ft.save("model_ft.h5")
 print('Time spent for model training is {} minutes'.format(round(( time.time() -start_time)/60,1)) )
x_train = regular_encode(training_set.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(validation_set.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test_set.content.values, tokenizer, maxlen=MAX_LEN)
 y_train = training_set.toxic.values
y_valid = validation_set.toxic.values<categorify>
yhat_val_cc = model_cc.predict(X_val_cc)
 print(yhat_val_cc )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
 )<create_dataframe>
yhat_val_ft = model_cc.predict(X_val_ft)
 print(yhat_val_ft )
test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<load_pretrained>
start_time = time.time()
test['Country_Region'] = test['Country_Region'].astype(str)
 test['Province_State'] = test['Province_State'].astype(str)
 country = test['Country_Region'].drop_duplicates()
adj_input_cols = [e for e in input_cols if e not in('province_encoder', 'country_encoder', 'date_int')]
test_df = test.copy().join(pd.DataFrame(columns = adj_input_cols + output_cols))
test_df.rename(columns={'Date': 'date', 'Country_Region': 'country', 'Province_State': 'province'}, inplace=True)
 lags = np.arange(1,8,1)
 test_overlap_mask =(test_df['date'] <= train_df['date'].max())
 train_overlap_mask =(train_df['date'] >= test_df['date'].min())
 test_df.loc[test_overlap_mask, input_cols + output_cols] = train_df.loc[train_overlap_mask, input_cols + output_cols].values
pred_dt_range = pd.date_range(start = train_df['date'].max() + pd.Timedelta(days=1), end = test_df['date'].max() , freq = '1D')
 with tqdm(total = len(pred_dt_range)) as pbar:
for d in pred_dt_range:
for i in country:
province = test_df[test_df['country'] == i]['province'].drop_duplicates()
for j in province:
mask =(test_df['date'] == d)&(test_df['country'] == i)&(test_df['province'] == j)
 for lag in lags:
mask_org =(test_df['date'] ==(d - pd.Timedelta(days=lag)))&(test_df['country'] == i)&(test_df['province'] == j)
 try:
test_df.loc[mask, 'cc_cases_' + str(lag)] = test_df.loc[mask_org, 'cc_cases'].values
except:
test_df.loc[mask, 'cc_cases_' + str(lag)] = 0
try:
test_df.loc[mask, 'ft_cases_' + str(lag)] = test_df.loc[mask_org, 'ft_cases'].values
except:
test_df.loc[mask, 'ft_cases_' + str(lag)] = 0
test_X = test_df.loc[mask, input_cols]
test_X_cc = test_X[cc_input]
X_test_cc= test_X_cc
X_test_cc = X_test_cc.to_numpy().reshape(X_test_cc.shape[0], 1, X_test_cc.shape[1])
 next_cc = model_cc.predict(X_test_cc)
 next_cc_scaled = next_cc
test_X_ft = test_X[ft_input]
X_test_ft = test_X_ft
X_test_ft = X_test_ft.to_numpy().reshape(X_test_ft.shape[0], 1, X_test_ft.shape[1])
 next_ft = model_cc.predict(X_test_ft)
 next_ft_scaled = next_ft
test_df.loc[mask, 'cc_cases'] = next_cc_scaled
test_df.loc[mask, 'ft_cases'] = next_ft_scaled
pbar.update(1)
 print('Time spent for building features is {} minutes'.format(round(( time.time() -start_time)/60,1)) )
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()<train_model>
submission = pd.DataFrame()
submission['ForecastId'] = test_df['ForecastId']
submission['ConfirmedCases'] = test_df['cc_cases']
submission['Fatalities'] = test_df['ft_cases']
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)<train_model>
submission.to_csv("submission.csv",index=False )