Kaggle competition: COVID19 Global Forecasting (Week 4)
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
warnings.filterwarnings("ignore")
 df_train=pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 df_test=pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 df_sub=pd.read_csv(".. /input/covid19-global-forecasting-week-4/submission.csv")
 print(df_train.shape)
 print(df_test.shape)
 print(df_sub.shape)
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<choose_model_class>
print(f"Unique Countries: {len(df_train.Country_Region.unique())}")
STRIDE_1 = 2
STRIDE_2 = 4
STRIDE_3 = 8
FILTER_1 = 64
FILTER_2 = 64
FILTER_3 = 64
inp = Input(shape=(maxlen,))
embed_layer1 = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 embed_layer1 = SpatialDropout1D(0.4 )(embed_layer1)
 line1 = Conv1D(FILTER_1, STRIDE_1 )(embed_layer1)
 line1 = Activation(LeakyReLU() )(line1)
 line1 = MaxPooling1D(STRIDE_1 )(line1)
 line1 = Conv1D(FILTER_1, STRIDE_1 )(line1)
 line1 = Activation(LeakyReLU() )(line1)
 line1 = MaxPooling1D(STRIDE_1 )(line1)
 line1 = Conv1D(FILTER_1, STRIDE_1 )(line1)
 line1 = Activation(LeakyReLU() )(line1)
 line1 = MaxPooling1D(STRIDE_1*2 )(line1)
 line1 = Flatten()(line1)
 line2 = Conv1D(FILTER_2, STRIDE_1 )(embed_layer1)
 line2 = Activation(LeakyReLU() )(line2)
 line2 = MaxPooling1D(STRIDE_1 )(line2)
 line2 = Conv1D(FILTER_2, STRIDE_1 )(line2)
 line2 = Activation(LeakyReLU() )(line2)
 line2 = MaxPooling1D(STRIDE_1 )(line2)
 line2 = Conv1D(FILTER_2, STRIDE_1 )(line2)
 line2 = Activation(LeakyReLU() )(line2)
 line2 = MaxPooling1D(STRIDE_1*2 )(line2)
 line2 = Flatten()(line2)
 line3 = Conv1D(FILTER_3, STRIDE_1 )(embed_layer1)
 line3 = Activation(LeakyReLU() )(line3)
 line3 = MaxPooling1D(STRIDE_1 )(line3)
 line3 = Conv1D(FILTER_3, STRIDE_1 )(line3)
 line3 = Activation(LeakyReLU() )(line3)
 line3 = MaxPooling1D(STRIDE_1 )(line3)
 line3 = Conv1D(FILTER_3, STRIDE_1 )(line3)
 line3 = Activation(LeakyReLU() )(line3)
 line3 = MaxPooling1D(STRIDE_1*2 )(line3)
 line3 = Flatten()(line3)
 concat_layer = concatenate([line1, line2, line3])
 conv1d_dense = Dense(1024, activation='relu' )(concat_layer)
 embed_layer2 = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 embed_layer2 = SpatialDropout1D(0.4 )(embed_layer2)
 rnn_line = Bidirectional(CuDNNLSTM(64, return_sequences=True), input_shape=(maxlen, embed_size))(embed_layer2)
 rnn_line = Bidirectional(CuDNNLSTM(64))(rnn_line)
 rnn_dense = Dense(1024, activation='relu' )(rnn_line)
 filter_sizes = [1,2,3,5]
num_filters = 36
embed_layer3 = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 embed_layer3 = SpatialDropout1D(0.4 )(embed_layer3)
 x = SpatialDropout1D(0.4 )(embed_layer3)
 x = Reshape(( maxlen, embed_size, 1))(x)
 conv_0 = Conv2D(num_filters, kernel_size=(filter_sizes[0], embed_size),
kernel_initializer='he_normal', activation='elu' )(x)
 conv_1 = Conv2D(num_filters, kernel_size=(filter_sizes[1], embed_size),
kernel_initializer='he_normal', activation='elu' )(x)
 conv_2 = Conv2D(num_filters, kernel_size=(filter_sizes[2], embed_size),
kernel_initializer='he_normal', activation='elu' )(x)
 conv_3 = Conv2D(num_filters, kernel_size=(filter_sizes[3], embed_size),
kernel_initializer='he_normal', activation='elu' )(x)
 maxpool_0 = MaxPool2D(pool_size=(maxlen - filter_sizes[0] + 1, 1))(conv_0)
 maxpool_1 = MaxPool2D(pool_size=(maxlen - filter_sizes[1] + 1, 1))(conv_1)
 maxpool_2 = MaxPool2D(pool_size=(maxlen - filter_sizes[2] + 1, 1))(conv_2)
 maxpool_3 = MaxPool2D(pool_size=(maxlen - filter_sizes[3] + 1, 1))(conv_3)
 cnn2d_dense = Concatenate(axis=1 )([maxpool_0, maxpool_1, maxpool_2, maxpool_3])
 cnn2d_dense = Flatten()(cnn2d_dense)
 cnn2d_dense = Dropout(0.1 )(cnn2d_dense)
 total = concatenate([cnn2d_dense, conv1d_dense, rnn_dense])
 preds = Dense(1, activation='sigmoid' )(total)
 model = Model(inputs=inp, outputs=preds )<choose_model_class>
print(f"Unique Regions: {df_train.shape[0]/len(df_train.Date.unique())}")
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy', f1] )<train_model>
df_train.Country_Region.value_counts()
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
print(f"Number of rows without Country_Region : {df_train.Country_Region.isna().sum() }")
pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))))<predict_on_test>
df_train["UniqueRegion"]=df_train.Country_Region
df_train.UniqueRegion[df_train.Province_State.isna() ==False]=df_train.Province_State+" , "+df_train.Country_Region
df_train[df_train.Province_State.isna() ==False]
pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<set_options>
df_train.drop(labels=["Id","Province_State","Country_Region"], axis=1, inplace=True)
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )<statistical_test>
test_dates=list(df_test.Date.unique())
 print(f"Period :{len(df_test.Date.unique())} days")
 print(f"From : {df_test.Date.min() } To : {df_test.Date.max() }" )
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<choose_model_class>
print(f"Total Regions : {df_test.shape[0]/43}" )
STRIDE_1 = 2
STRIDE_2 = 4
STRIDE_3 = 8
FILTER_1 = 64
FILTER_2 = 64
FILTER_3 = 64
inp = Input(shape=(maxlen,))
embed_layer1 = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 embed_layer1 = SpatialDropout1D(0.4 )(embed_layer1)
 line1 = Conv1D(FILTER_1, STRIDE_1 )(embed_layer1)
 line1 = Activation(LeakyReLU() )(line1)
 line1 = MaxPooling1D(STRIDE_1 )(line1)
 line1 = Conv1D(FILTER_1, STRIDE_1 )(line1)
 line1 = Activation(LeakyReLU() )(line1)
 line1 = MaxPooling1D(STRIDE_1 )(line1)
 line1 = Conv1D(FILTER_1, STRIDE_1 )(line1)
 line1 = Activation(LeakyReLU() )(line1)
 line1 = MaxPooling1D(STRIDE_1*2 )(line1)
 line1 = Flatten()(line1)
 line2 = Conv1D(FILTER_2, STRIDE_1 )(embed_layer1)
 line2 = Activation(LeakyReLU() )(line2)
 line2 = MaxPooling1D(STRIDE_1 )(line2)
 line2 = Conv1D(FILTER_2, STRIDE_1 )(line2)
 line2 = Activation(LeakyReLU() )(line2)
 line2 = MaxPooling1D(STRIDE_1 )(line2)
 line2 = Conv1D(FILTER_2, STRIDE_1 )(line2)
 line2 = Activation(LeakyReLU() )(line2)
 line2 = MaxPooling1D(STRIDE_1*2 )(line2)
 line2 = Flatten()(line2)
 line3 = Conv1D(FILTER_3, STRIDE_1 )(embed_layer1)
 line3 = Activation(LeakyReLU() )(line3)
 line3 = MaxPooling1D(STRIDE_1 )(line3)
 line3 = Conv1D(FILTER_3, STRIDE_1 )(line3)
 line3 = Activation(LeakyReLU() )(line3)
 line3 = MaxPooling1D(STRIDE_1 )(line3)
 line3 = Conv1D(FILTER_3, STRIDE_1 )(line3)
 line3 = Activation(LeakyReLU() )(line3)
 line3 = MaxPooling1D(STRIDE_1*2 )(line3)
 line3 = Flatten()(line3)
 concat_layer = concatenate([line1, line2, line3])
 conv1d_dense = Dense(1024, activation='relu' )(concat_layer)
 embed_layer2 = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 embed_layer2 = SpatialDropout1D(0.4 )(embed_layer2)
 rnn_line = Bidirectional(CuDNNLSTM(64, return_sequences=True), input_shape=(maxlen, embed_size))(embed_layer2)
 rnn_line = Bidirectional(CuDNNLSTM(64))(rnn_line)
 rnn_dense = Dense(1024, activation='relu' )(rnn_line)
 filter_sizes = [1,2,3,5]
num_filters = 36
embed_layer3 = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 embed_layer3 = SpatialDropout1D(0.4 )(embed_layer3)
 x = SpatialDropout1D(0.4 )(embed_layer3)
 x = Reshape(( maxlen, embed_size, 1))(x)
 conv_0 = Conv2D(num_filters, kernel_size=(filter_sizes[0], embed_size),
kernel_initializer='he_normal', activation='elu' )(x)
 conv_1 = Conv2D(num_filters, kernel_size=(filter_sizes[1], embed_size),
kernel_initializer='he_normal', activation='elu' )(x)
 conv_2 = Conv2D(num_filters, kernel_size=(filter_sizes[2], embed_size),
kernel_initializer='he_normal', activation='elu' )(x)
 conv_3 = Conv2D(num_filters, kernel_size=(filter_sizes[3], embed_size),
kernel_initializer='he_normal', activation='elu' )(x)
 maxpool_0 = MaxPool2D(pool_size=(maxlen - filter_sizes[0] + 1, 1))(conv_0)
 maxpool_1 = MaxPool2D(pool_size=(maxlen - filter_sizes[1] + 1, 1))(conv_1)
 maxpool_2 = MaxPool2D(pool_size=(maxlen - filter_sizes[2] + 1, 1))(conv_2)
 maxpool_3 = MaxPool2D(pool_size=(maxlen - filter_sizes[3] + 1, 1))(conv_3)
 cnn2d_dense = Concatenate(axis=1 )([maxpool_0, maxpool_1, maxpool_2, maxpool_3])
 cnn2d_dense = Flatten()(cnn2d_dense)
 cnn2d_dense = Dropout(0.1 )(cnn2d_dense)
 total = concatenate([cnn2d_dense, conv1d_dense, rnn_dense])
 preds = Dense(1, activation='sigmoid' )(total)
 model = Model(inputs=inp, outputs=preds )<choose_model_class>
df_test["UniqueRegion"]=df_test.Country_Region
df_test.UniqueRegion[df_test.Province_State.isna() ==False]=df_test.Province_State+" , "+df_test.Country_Region
df_test.drop(labels=["Province_State","Country_Region"], axis=1, inplace=True )
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy', f1] )<train_model>
len(df_test.UniqueRegion.unique() )
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))<predict_on_test>
only_train_dates=set(train_dates)-set(test_dates)
 print("Only train dates : ",len(only_train_dates))
intersection_dates=set(test_dates)&set(train_dates)
 print("Intersection dates : ",len(intersection_dates))
only_test_dates=set(test_dates)-set(train_dates)
 print("Only Test dates : ",len(only_test_dates))
pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))))<predict_on_test>
df_test_temp=pd.DataFrame()
df_test_temp["Date"]=df_test.Date
df_test_temp["ConfirmedCases"]=0.0
df_test_temp["Fatalities"]=0.0
df_test_temp["UniqueRegion"]=df_test.UniqueRegion
df_test_temp["Delta"]=1.0
pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1 )<compute_test_metric>
%%time
final_df=pd.DataFrame(columns=["Date","ConfirmedCases","Fatalities","UniqueRegion"])
 for region in df_train.UniqueRegion.unique() :
df_temp=df_train[df_train.UniqueRegion==region].reset_index()
df_temp["Delta"]=1.0
size_train=df_temp.shape[0]
for i in range(1,df_temp.shape[0]):
if(df_temp.ConfirmedCases[i-1]>0):
df_temp.Delta[i]=df_temp.ConfirmedCases[i]/df_temp.ConfirmedCases[i-1]
n=5
delta_list=df_temp.tail(n ).Delta
delta_avg=df_temp.tail(n ).Delta.mean()
death_rate=df_temp.tail(1 ).Fatalities.sum() /df_temp.tail(1 ).ConfirmedCases.sum()
df_test_app=df_test_temp[df_test_temp.UniqueRegion==region]
df_test_app=df_test_app[df_test_app.Date>df_temp.Date.max() ]
X=np.arange(1,n+1 ).reshape(-1,1)
 Y=delta_list
model=LinearRegression()
model.fit(X,Y)
 df_temp=pd.concat([df_temp,df_test_app])
 df_temp=df_temp.reset_index()
for i in range(size_train, df_temp.shape[0]):
n=n+1
df_temp.Delta[i]=max(1,model.predict(np.array([n] ).reshape(-1,1)) [0])
 for i in range(size_train, df_temp.shape[0]):
df_temp.ConfirmedCases[i]=round(df_temp.ConfirmedCases[i-1]*df_temp.Delta[i],0)
 df_temp.Fatalities[i]=round(death_rate*df_temp.ConfirmedCases[i],0)
 size_test=df_temp.shape[0]-df_test_temp[df_test_temp.UniqueRegion==region].shape[0]
df_temp=df_temp.iloc[size_test:,:]
df_temp=df_temp[["Date","ConfirmedCases","Fatalities","UniqueRegion","Delta"]]
final_df=pd.concat([final_df,df_temp], ignore_index=True)
 final_df.shape
<save_to_csv><EOS> 
df_sub.Fatalities=final_df.Fatalities
df_sub.ConfirmedCases=final_df.ConfirmedCases
df_sub.to_csv("submission.csv", index=None )