Kaggle competition: COVID19 Global Forecasting (Week 4)
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )<split>
plt.style.use('dark_background')
 warnings.filterwarnings('ignore')
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)
 embed_size = 300
max_features = 60000
maxlen = 120
train_X = train_df["question_text"].fillna("_na_" ).values
val_X = val_df["question_text"].fillna("_na_" ).values
test_X = test_df["question_text"].fillna("_na_" ).values
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values<randomize_order>
train_data=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 test_data=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv' )
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[trn_idx]
val_X = val_X[val_idx]
train_y = train_y[trn_idx]
val_y = val_y[val_idx]<train_model>
train_data.rename(columns={'Country_Region':'Country'}, inplace=True)
 test_data.rename(columns={'Country_Region':'Country'}, inplace=True)
 train_data.rename(columns={'Province_State':'Province'}, inplace=True)
 test_data.rename(columns={'Province_State':'Province'}, inplace=True )
<set_options>
train_data['Date'] = pd.to_datetime(train_data['Date'])
 test_data['Date'] = pd.to_datetime(test_data['Date'] )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<statistical_test>
train_data['Date']=train_data['Date'].dt.strftime('%m%d')
 test_data['Date']=test_data['Date'].dt.strftime('%m%d')
 train_data['Date']=train_data['Date'].astype(int)
 test_data['Date']=test_data['Date'].astype(int )
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )<train_model>
train_data["Province"]=train_data.apply(lambda row: str(row['Country'])if pd.isnull(row['Province'])else row['Province'] ,axis=1)
 test_data["Province"]=test_data.apply(lambda row: str(row['Country'])if pd.isnull(row['Province'])else row['Province'] ,axis=1 )
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))<predict_on_test>
encoder=LabelEncoder()
train_data['Province']=encoder.fit_transform(train_data['Province'])
 test_data['Province']=encoder.fit_transform(test_data['Province'])
 train_data['Country']=encoder.fit_transform(train_data['Country'])
 test_data['Country']=encoder.fit_transform(test_data['Country'] )
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))<predict_on_test>
train_data.drop(['Id'],axis=1,inplace=True )
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
features=['Province','Country','Date']
target_1=['ConfirmedCases']
target_2=['Fatalities']
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
scaler = MinMaxScaler()
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )<train_model>
train_data[target_1]=train_data[target_1].astype(int)
 train_data[target_2]=train_data[target_2].astype(int )
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))<predict_on_test>
xgb_clf_1=XGBRegressor(n_estimators=1300)
 xgb_clf_2=XGBRegressor(n_estimators=1300 )
pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))))<predict_on_test>
xgb_df=pd.DataFrame({'ForecastId':[],'ConfirmedCases':[],'Fatalities':[]})
 for country in train_data['Country'].unique() :
Province=train_data.loc[train_data.Country == country, :].Province.unique().tolist()
for province in Province:
new_train=train_data.loc[train_data['Province']==province]
new_train[features] = scaler.fit_transform(new_train[features].values)
 xgb_clf_1.fit(new_train[features],new_train[target_1])
 xgb_clf_2.fit(new_train[features],new_train[target_2])
 new_test=test_data.loc[test_data['Province']==province]
new_test_FI=new_test.ForecastId
new_test[features]=scaler.transform(new_test[features])
 new_pred_1=xgb_clf_1.predict(new_test[features])
 new_pred_2=xgb_clf_2.predict(new_test[features])
 new_df=pd.DataFrame({'ForecastId':new_test_FI,'ConfirmedCases':new_pred_1,'Fatalities':new_pred_2})
 xgb_df=pd.concat([xgb_df,new_df],axis=0 )
pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
xgb_df=xgb_df.drop_duplicates()
xgb_df['ForecastId']=xgb_df['ForecastId'].astype(int)
 xgb_df['ConfirmedCases']=np.round(xgb_df['ConfirmedCases'])
 xgb_df['Fatalities']=np.round(xgb_df['Fatalities'] )
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
xgb_df.to_csv('submission.csv',index=False )