Kaggle competition: COVID19 Global Forecasting (Week 4)
<SOS> metric: MCRMSLE Kaggle data source: covid19-global-forecasting-(week-4)<compute_test_metric>
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
pd.set_option('display.max_columns', 99)
 pd.set_option('display.max_rows', 99)
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.decode().split(" ")) for o in embeddings )<import_modules>
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
sns.set_palette(sns.color_palette('tab20', 20))
import tensorflow as tf
from sklearn.model_selection import train_test_split<define_variables>
DATA_PATH = '.. /input/covid19-metadata'
DATEFORMAT = '%Y-%m-%d'
def get_comp_data(COMP):
train = pd.read_csv(f'{COMP}/train.csv')
 test = pd.read_csv(f'{COMP}/test.csv')
 submission = pd.read_csv(f'{COMP}/submission.csv')
 print(train.shape, test.shape, submission.shape)
 train['Country_Region'] = train['Country_Region'].str.replace(',', '')
 test['Country_Region'] = test['Country_Region'].str.replace(',', '')
 train['Location'] = train['Country_Region'] + '-' + train['Province_State'].fillna('')
 test['Location'] = test['Country_Region'] + '-' + test['Province_State'].fillna('')
 train['LogConfirmed'] = to_log(train.ConfirmedCases)
 train['LogFatalities'] = to_log(train.Fatalities)
 train = train.drop(columns=['Province_State'])
 test = test.drop(columns=['Province_State'])
 country_codes = pd.read_csv(f'{DATA_PATH}/country_codes.csv', keep_default_na=False)
 train = train.merge(country_codes, on='Country_Region', how='left')
 test = test.merge(country_codes, on='Country_Region', how='left')
 train['DateTime'] = pd.to_datetime(train['Date'])
 test['DateTime'] = pd.to_datetime(test['Date'])
 train = train.sort_values(by='Date')
 test = test.sort_values(by='Date')
 train = train.fillna('
test = test.fillna('
return train, test, submission
def process_each_location(df):
dfs = []
for loc, df in df.groupby('Location'):
df = df.sort_values(by='Date')
 df['Fatalities'] = df['Fatalities'].cummax()
df['ConfirmedCases'] = df['ConfirmedCases'].cummax()
df['LogFatalities'] = df['LogFatalities'].cummax()
df['LogConfirmed'] = df['LogConfirmed'].cummax()
df['LogConfirmedNextDay'] = df['LogConfirmed'].shift(-1)
 df['ConfirmedNextDay'] = df['ConfirmedCases'].shift(-1)
 df['DateNextDay'] = df['Date'].shift(-1)
 df['LogFatalitiesNextDay'] = df['LogFatalities'].shift(-1)
 df['FatalitiesNextDay'] = df['Fatalities'].shift(-1)
 df['LogConfirmedDelta'] = df['LogConfirmedNextDay'] - df['LogConfirmed']
df['ConfirmedDelta'] = df['ConfirmedNextDay'] - df['ConfirmedCases']
df['LogFatalitiesDelta'] = df['LogFatalitiesNextDay'] - df['LogFatalities']
df['FatalitiesDelta'] = df['FatalitiesNextDay'] - df['Fatalities']
dfs.append(df)
 return pd.concat(dfs)
 def add_days(d, k):
return dt.datetime.strptime(d, DATEFORMAT)+ dt.timedelta(days=k)
 def to_log(x):
return np.log(x + 1)
 def to_exp(x):
return np.exp(x)- 1
def create_features(train_set):
dfs = []
for loc, df in train_set.groupby('Location'):
df = df.sort_values(by='Date' ).copy()
df['f_lc_7d'] = df['LogConfirmed'].shift(7)
 df['f_lf_7d'] = df['LogFatalities'].shift(7)
 df['f_lc_3d'] = df['LogConfirmed'].shift(3)
 df['f_lf_3d'] = df['LogFatalities'].shift(3)
 df['f_lc_1d'] = df['LogConfirmed'].shift(1)
 df['f_lf_1d'] = df['LogFatalities'].shift(1)
 df['f_lc_0d'] = df['LogConfirmed']
df['f_lf_0d'] = df['LogFatalities']
df['f_fc_rate'] = np.clip(( to_exp(df['LogFatalities'])+ 1)/(to_exp(df['LogConfirmed'])+ 1), 0, 0.15)
 dfs.append(df)
 dfs = pd.concat(dfs)
 dfs['d_lc_7d'] = dfs['f_lc_0d'] - dfs['f_lc_7d']
dfs['d_lf_7d'] = dfs['f_lf_0d'] - dfs['f_lf_7d']
dfs['d_lc_3d'] = dfs['f_lc_3d'] - dfs['f_lc_3d']
dfs['d_lf_3d'] = dfs['f_lf_3d'] - dfs['f_lf_3d']
dfs['d_lc_1d'] = dfs['f_lc_0d'] - dfs['f_lc_1d']
dfs['d_lf_1d'] = dfs['f_lf_0d'] - dfs['f_lf_1d']
return dfs
stopwords = [ "a", "about", "above", "after", "again", "against", "all", "am", "an", "and", "any", "are", "as", "at", "be", "because", "been", "before", "being", "below", "between", "both", "but", "by", "could", "did", "do", "does", "doing", "down", "during", "each", "few", "for", "from", "further", "had", "has", "have", "having", "he", "he'd", "he'll", "he's", "her", "here", "here's", "hers", "herself", "him", "himself", "his", "how", "how's", "i", "i'd", "i'll", "i'm", "i've", "if", "in", "into", "is", "it", "it's", "its", "itself", "let's", "me", "more", "most", "my", "myself", "nor", "of", "on", "once", "only", "or", "other", "ought", "our", "ours", "ourselves", "out", "over", "own", "same", "she", "she'd", "she'll", "she's", "should", "so", "some", "such", "than", "that", "that's", "the", "their", "theirs", "them", "themselves", "then", "there", "there's", "these", "they", "they'd", "they'll", "they're", "they've", "this", "those", "through", "to", "too", "under", "until", "up", "very", "was", "we", "we'd", "we'll", "we're", "we've", "were", "what", "what's", "when", "when's", "where", "where's", "which", "while", "who", "who's", "whom", "why", "why's", "with", "would", "you", "you'd", "you'll", "you're", "you've", "your", "yours", "yourself", "yourselves" ] 
<load_from_csv>
COMP = '.. /input/covid19-global-forecasting-week-4'
start = dt.datetime.now()
train, test, submission = get_comp_data(COMP)
 train.shape, test.shape, submission.shape
train.head(2)
 test.head(2)
 TRAIN_START = train.Date.min()
TEST_START = test.Date.min()
TRAIN_END = train.Date.max()
TEST_END = test.Date.max()
print(TRAIN_START, TRAIN_END, TEST_START, TEST_END )
train_data = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')
 test_data = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv' )<define_variables>
train_clean = process_each_location(train)
 print('train cleaned', train_clean.shape)
 train_clean = train_clean[[
'Location', 'Date', 'continent',
'LogConfirmed', 'LogFatalities',
'LogConfirmedDelta', 'LogFatalitiesDelta'
]]
continent_encoder = preprocessing.LabelEncoder()
train_clean['f_continent'] = continent_encoder.fit_transform(train_clean.continent)
 train_features = create_features(train_clean)
 train_features.head()
print(train_data.columns)
 train_input = list(train_data['question_text'])
 train_label = list(train_data['target'])
 test_input = list(test_data['question_text'] )<string_transform>
def predict(min_child_weight, eta, colsample_bytree, max_depth, subsample,
NROUND, PRECISION, DECAY, WEIGHT_NORM, MIN_DATE):
train, test, submission = get_comp_data(COMP)
 train_clean = process_each_location(train)
 print('train cleaned', train_clean.shape)
 train_clean = train_clean[[
'Location', 'Date', 'continent',
'LogConfirmed', 'LogFatalities',
'LogConfirmedDelta', 'LogFatalitiesDelta'
]]
continent_encoder = preprocessing.LabelEncoder()
train_clean['f_continent'] = continent_encoder.fit_transform(train_clean.continent)
 train_features = create_features(train_clean)
 VALID_CUTOFF = TRAIN_END
features = [
'f_lc_7d', 'f_lf_7d', 'f_lc_3d', 'f_lf_3d', 'f_continent',
'f_lc_1d', 'f_lf_1d', 'f_lc_0d', 'f_lf_0d', 'f_fc_rate'
]
diff_features = ['d_lc_7d', 'd_lf_7d', 'd_lc_3d', 'd_lf_3d', 'd_lc_1d', 'd_lf_1d']
features = features + diff_features
print(f'{len(features)} features: ')
 print(features)
 fix = {
'lambda': 1., 'nthread': 3, 'booster': 'gbtree',
'silent': 1, 'eval_metric': 'rmse',
'objective': 'reg:squarederror'}
config = dict(
 min_child_weight=min_child_weight,
eta=eta, colsample_bytree=colsample_bytree,
max_depth=max_depth, subsample=subsample)
 config.update(fix)
 Xtr = train_features[(train_features.Date >= MIN_DATE)&(train_features.Date < VALID_CUTOFF)].copy()
Xtr['days'] = -(pd.to_datetime(train_features.Date)- dt.datetime.strptime(VALID_CUTOFF, DATEFORMAT)).dt.days
print(Xtr.shape)
 print(config)
 print(PRECISION, NROUND, DECAY, WEIGHT_NORM)
 def weighting(days):
return 1./ days ** WEIGHT_NORM
dtrain_lc = xgb.DMatrix(Xtr[features].round(PRECISION), label=Xtr.LogConfirmedDelta, weight=weighting(Xtr.days))
dtrain_lf = xgb.DMatrix(Xtr[features].round(PRECISION), label=Xtr.LogFatalitiesDelta, weight=weighting(Xtr.days))
model_lc = xgb.train(config, dtrain_lc, NROUND, evals=[(dtrain_lc, 'train-lc')], verbose_eval=100)
 model_lf = xgb.train(config, dtrain_lf, NROUND, evals=[(dtrain_lf, 'train-lf')], verbose_eval=100)
 predictions = Xtr.copy()
predictions = train_features[(train_features.Date >= MIN_DATE)&(train_features.Date <= VALID_CUTOFF)].copy()
predictions.LogConfirmedDelta = np.nan
predictions.LogFatalitiesDelta = np.nan
for i, d in enumerate(pd.date_range(VALID_CUTOFF, add_days(TEST_END, 1))):
last_day = str(d ).split(' ')[0]
next_day = dt.datetime.strptime(last_day, DATEFORMAT)+ dt.timedelta(days=1)
 next_day = next_day.strftime(DATEFORMAT)
 p_next_day = predictions[predictions.Date == last_day].copy()
p_next_day.Date = next_day
p_next_day['plc'] = model_lc.predict(xgb.DMatrix(p_next_day[features].round(PRECISION)))
 p_next_day['plf'] = model_lf.predict(xgb.DMatrix(p_next_day[features].round(PRECISION)))
 p_next_day.LogConfirmed = p_next_day.LogConfirmed + np.clip(p_next_day['plc'], 0, None)* DECAY ** i
p_next_day.LogFatalities = p_next_day.LogFatalities + np.clip(p_next_day['plf'], 0, None)* DECAY ** i
predictions = pd.concat([predictions, p_next_day], sort=True)
 predictions = create_features(predictions)
 predictions['PC'] = to_exp(predictions.LogConfirmed)
 predictions['PF'] = to_exp(predictions.LogFatalities)
 return predictions
def remove_stop_words(x):
for word in stopwords:
token = " " + word + " "
if(x.find(token)!= -1):
x = x.replace(token, " ")
 return x
train_input_rsw = list(map(remove_stop_words, train_input))
test_input_rsw = list(map(remove_stop_words, test_input))<train_model>
decay = 0.99
prediction_1 = predict(min_child_weight=5, eta=0.01, colsample_bytree=0.8, max_depth=5, subsample=0.9,
NROUND=800, PRECISION=2, DECAY=decay, WEIGHT_NORM=0.25, MIN_DATE='2020-03-22')
 prediction_2 = predict(min_child_weight=7, eta=0.01, colsample_bytree=0.7, max_depth=6, subsample=0.8,
NROUND=1000, PRECISION=2, DECAY=decay, WEIGHT_NORM=0.23, MIN_DATE='2020-03-15')
 prediction_3 = predict(min_child_weight=3, eta=0.01, colsample_bytree=0.6, max_depth=7, subsample=0.7,
NROUND=1200, PRECISION=2, DECAY=decay, WEIGHT_NORM=0.2, MIN_DATE='2020-03-08')
 prediction_4 = predict(min_child_weight=3, eta=0.011, colsample_bytree=0.75, max_depth=10, subsample=0.6,
NROUND=1200, PRECISION=3, DECAY=decay, WEIGHT_NORM=0.15, MIN_DATE='2020-03-15')
 prediction_5 = predict(min_child_weight=10, eta=0.008, colsample_bytree=0.75, max_depth=10, subsample=0.6,
NROUND=1500, PRECISION=3, DECAY=decay, WEIGHT_NORM=0.2, MIN_DATE='2020-03-22')
 prediction_6 = predict(min_child_weight=20, eta=0.01, colsample_bytree=0.7, max_depth=5, subsample=0.85,
NROUND=1000, PRECISION=3, DECAY=decay, WEIGHT_NORM=0.3, MIN_DATE='2020-03-22' )
tokenizer = Tokenizer()
tokenizer.fit_on_texts(train_input_rsw)
 word_index = tokenizer.word_index
embedding_matrix = np.zeros(( len(word_index)+1, 300))
for word, index in word_index.items() :
embedding_vector = embeddings_index.get(word)
 if(embedding_vector is not None):
embedding_matrix[index] = embedding_vector
print(embedding_matrix.shape )<set_options>
cols = ['Location', 'Date', 'PC', 'PF']
p12 = pd.merge(prediction_1[cols], prediction_2[cols], on=['Location', 'Date'], suffixes=['_1', '_2'])
 p34 = pd.merge(prediction_3[cols], prediction_4[cols], on=['Location', 'Date'], suffixes=['_3', '_4'])
 p56 = pd.merge(prediction_5[cols], prediction_6[cols], on=['Location', 'Date'], suffixes=['_5', '_6'])
 preds = pd.merge(p12, p34, on=['Location', 'Date'])
 preds = pd.merge(preds, p56, on=['Location', 'Date'])
 preds.head()
c = preds.loc[preds.Date >= '2020-04-15'].corr()
c
fig = px.imshow(c)
 fig.show()
del embeddings_index
gc.collect()<define_variables>
pcs = ['PC_1', 'PC_2', 'PC_3', 'PC_4', 'PC_5', 'PC_6']
pfs = ['PF_1', 'PF_2', 'PF_3', 'PF_4', 'PF_5', 'PF_6']
preds['PC'] = to_exp(to_log(preds[pcs] ).mean(axis=1))
preds['PF'] = to_exp(to_log(preds[pfs] ).mean(axis=1))
preds.tail()
max_length = 30<string_transform>
my_submission = test.copy()
my_submission = my_submission.merge(preds)
 my_submission['ConfirmedCases'] = my_submission['PC']
my_submission['Fatalities'] = my_submission['PF']
my_submission.shape
my_submission.head()
sequences = tokenizer.texts_to_sequences(train_input_rsw)
 train_input_padded = pad_sequences(sequences, maxlen=max_length, padding='post', truncating='post')
 print(train_input_padded.shape)
 sequences = tokenizer.texts_to_sequences(test_input_rsw)
 test_input_padded = pad_sequences(sequences, maxlen=max_length, padding='post', truncating='post')
 print(test_input_padded.shape )<split>
my_submission[[
'ForecastId', 'ConfirmedCases', 'Fatalities'
]].to_csv('submission.csv', index=False)
 my_submission.groupby('Date')[['ConfirmedCases', 'Fatalities']].sum().reset_index().tail()
<choose_model_class><EOS> 
end = dt.datetime.now()
print('Finished', end,(end - start ).seconds, 's' )