Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<define_variables>
import numpy as np
import pandas as pd
import seaborn as sns
from sklearn.model_selection import train_test_split
from xgboost import XGBRegressor
from sklearn.multioutput import MultiOutputRegressor
from sklearn.impute import SimpleImputer
VAL_Y = []
TEST_Y = []<statistical_test>
train_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 submission_csv = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv' )
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<train_model>
train_data['Date'] = pd.to_datetime(train_data['Date'], infer_datetime_format=True)
 test_data['Date'] = pd.to_datetime(test_data['Date'], infer_datetime_format=True )
model = model_cnn(embedding_matrix)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 batch_size = 2048
epochs = 5
model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)
 model.save('./model_cnn_glove.h5' )<predict_on_test>
train_data.loc[:, 'Date'] = train_data.Date.dt.strftime('%y%m%d')
 train_data.loc[:, 'Date'] = train_data['Date'].astype(int)
 test_data.loc[:, 'Date'] = test_data.Date.dt.strftime('%y%m%d')
 test_data.loc[:, 'Date'] = test_data['Date'].astype(int )
val_pred_cnn_glove = model.predict([X_val], batch_size=1024, verbose=1)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(val_pred_cnn_glove > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh)
 y_pred_cnn_glove = model.predict(x_test, batch_size=1024, verbose=True)
 VAL_Y.append(val_pred_cnn_glove)
 TEST_Y.append(y_pred_cnn_glove )<train_model>
train_data['Province_State'] = np.where(train_data['Province_State'] == 'nan',train_data['Country_Region'],train_data['Province_State'])
 test_data['Province_State'] = np.where(test_data['Province_State'] == 'nan',test_data['Country_Region'],test_data['Province_State'] )
model = model_bgru(embedding_matrix)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 batch_size = 2048
epochs = 5
model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)
 model.save('./model_bgru_glove.h5' )<predict_on_test>
convert_dict = {'Province_State': str}
train_data = train_data.astype(convert_dict)
 test_data = test_data.astype(convert_dict )
val_pred_bgru_glove = model.predict([X_val], batch_size=1024, verbose=1)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(val_pred_bgru_glove > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh)
 y_pred_bgru_glove = model.predict(x_test, batch_size=1024, verbose=True)
 VAL_Y.append(val_pred_bgru_glove)
 TEST_Y.append(y_pred_bgru_glove )<drop_column>
s =(train_data.dtypes == 'object')
 object_cols = list(s[s].index )
del embeddings_index;
del embedding_matrix
gc.collect()<statistical_test>
from sklearn.preprocessing import LabelEncoder
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<train_model>
label_encoder1 = LabelEncoder()
label_encoder2 = LabelEncoder()
train_data['Province_State'] = label_encoder1.fit_transform(train_data['Province_State'])
 test_data['Province_State'] = label_encoder1.transform(test_data['Province_State'])
 train_data['Country_Region'] = label_encoder2.fit_transform(train_data['Country_Region'])
 test_data['Country_Region'] = label_encoder2.transform(test_data['Country_Region'])
model = model_bgru(embedding_matrix)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 batch_size = 2048
epochs = 5
model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)
 model.save('./model_bgru_wiki.h5' )<predict_on_test>
Test_id = test_data.ForecastId
val_pred_bgru_wiki = model.predict([X_val], batch_size=1024, verbose=1)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(val_pred_bgru_wiki > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh)
 y_pred_bgru_wiki = model.predict(x_test, batch_size=1024, verbose=True)
 VAL_Y.append(val_pred_bgru_wiki)
 TEST_Y.append(y_pred_bgru_wiki )<drop_column>
train_data.drop(['Id'], axis=1, inplace=True)
 test_data.drop('ForecastId', axis=1, inplace=True )
del embeddings_index;
del embedding_matrix
gc.collect()<statistical_test>
missing_val_count_by_column =(train_data.isnull().sum())
 print(missing_val_count_by_column[missing_val_count_by_column>0] )
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<train_model>
from xgboost import XGBRegressor
model = model_bgru(embedding_matrix)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])
 batch_size = 2048
epochs = 5
model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)
 model.save('./model_bgru_paragram.h5' )<predict_on_test>
X_train = train_data[['Province_State','Country_Region','Date']]
y_train = train_data[['ConfirmedCases', 'Fatalities']]
val_pred_bgru_paragram = model.predict([X_val], batch_size=1024, verbose=1)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(val_pred_bgru_paragram > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh)
 y_pred_bgru_paragram = model.predict(x_test, batch_size=1024, verbose=True)
 VAL_Y.append(val_pred_bgru_paragram)
 TEST_Y.append(y_pred_bgru_paragram )<compute_test_metric>
y_train_confirm = y_train.ConfirmedCases
y_train_fatality = y_train.Fatalities
pred_val_y =(2.5*val_pred_cnn_glove + 2.5*val_pred_bgru_glove + 2.5*val_pred_bgru_wiki + 2.5*val_pred_bgru_paragram)/10
thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(y_val,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh )<save_to_csv>
x_train = X_train.iloc[:,:].values
x_test = X_train.iloc[:,:].values
y_pred =(2.5*y_pred_cnn_glove + 2.5*y_pred_bgru_glove + 2.5*y_pred_bgru_wiki + 2.5*y_pred_bgru_paragram)/10
y_te =(y_pred[:,0] > best_thresh ).astype(np.int)
 submit_df = pd.DataFrame({"qid": test_df["qid"], "prediction": y_te})
 submit_df.to_csv("submission.csv", index=False )<save_to_csv>
model1 = XGBRegressor(n_estimators=40000)
 model1.fit(X_train, y_train_confirm)
 y_pred_confirm = model1.predict(test_data )
def create_download_link(df, title = "Download CSV file", filename = "data.csv"):
csv = df.to_csv(index =False)
 b64 = base64.b64encode(csv.encode())
 payload = b64.decode()
html = '<a download="{filename}" href="data:text/csv;base64,{payload}" target="_blank">{title}</a>'
html = html.format(payload=payload,title=title,filename=filename)
 return HTML(html)
 create_download_link(submit_df )<import_modules>
model2 = XGBRegressor(n_estimators=20000)
 model2.fit(X_train,y_train_fatality)
 y_pred_fat = model2.predict(test_data )
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers<load_from_csv>
df_sub = pd.DataFrame()
df_sub['ForecastId'] = Test_id
df_sub['ConfirmedCases'] = y_pred_confirm
df_sub['Fatalities'] = y_pred_fat
df_sub.to_csv('submission.csv', index=False )