Kaggle competition: COVID19 Global Forecasting (Week 1)
%matplotlib inline
%matplotlib inline 
<define_variables>
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/train.csv')
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/test.csv')
 submission = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/submission.csv' )
path = '.. /input/'
comp = 'jigsaw-toxic-comment-classification-challenge/'
TRAIN_DATA_FILE=f'{path}{comp}train.csv.zip'
TEST_DATA_FILE=f'{path}{comp}test.csv.zip'
TEST_LABELS_FILE = f'{path}{comp}test_labels.csv.zip'<load_from_csv>
test['Date']=test.Date.astype('datetime64[ns]' )
train = pd.read_csv(TRAIN_DATA_FILE)
 test = pd.read_csv(TEST_DATA_FILE)
 test_l = pd.read_csv(TEST_LABELS_FILE)
 sample_submission = pd.read_csv(f'{path}{comp}sample_submission.csv.zip' )<define_variables>
train['key']=train['Province/State'].astype('str')+ " " + train['Country/Region'].astype('str')+ " " +train['Lat'].astype('str')+ " " +train['Long'].astype('str' )
embed_size = 300
max_features = 20000
maxlen = 200<count_missing_values>
test['key']=test['Province/State'].astype('str')+ " " + test['Country/Region'].astype('str')+ " " +test['Lat'].astype('str')+ " " +test['Long'].astype('str' )
train.isnull().any() ,test.isnull().any()<categorify>
daily_analysis=train.groupby(['Date'] ).sum()
def clean_text(text):
text = text.lower()
text = re.sub(r"what's", "what is ", text)
 text = re.sub(r"'s", " ", text)
 text = re.sub(r"'ve", " have ", text)
 text = re.sub(r"can't", "cannot ", text)
 text = re.sub(r"n't", " not ", text)
 text = re.sub(r"i'm", "i am ", text)
 text = re.sub(r"'re", " are ", text)
 text = re.sub(r"'d", " would ", text)
 text = re.sub(r"'ll", " will ", text)
 text = re.sub(r"'scuse", " excuse ", text)
 text = re.sub('\W', ' ', text)
 text = re.sub('\s+', ' ', text)
 text = re.sub(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', ' _ip_ ', text)
 text = re.sub(r'(['"\.\(\)\!\?\-\\\/\,])', r' \1 ',text)
 text = re.sub(r'([\;\:\|•«
])', ' ', text)
 text = text.replace('&', ' and ')
 text = text.replace('@', ' at ')
 text = text.replace('0', ' zero ')
 text = text.replace('1', ' one ')
 text = text.replace('2', ' two ')
 text = text.replace('3', ' three ')
 text = text.replace('4', ' four ')
 text = text.replace('5', ' five ')
 text = text.replace('6', ' six ')
 text = text.replace('7', ' seven ')
 text = text.replace('8', ' eight ')
 text = text.replace('9', ' nine ')
 text = text.strip(' ')
 return text<feature_engineering>
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)) )
train['comment_text'] = train['comment_text'].map(lambda com : clean_text(com))<prepare_x_and_y>
train_lag_1=train.groupby(['key'] ).shift(periods=1)
 train_lag_2=train.groupby(['key'] ).shift(periods=2)
 train_lag_3=train.groupby(['key'] ).shift(periods=3)
 train_lag_4=train.groupby(['key'] ).shift(periods=4)
 train['lag_1_ConfirmedCases']=train_lag_1['ConfirmedCases']
train['lag_1_Fatalities']=train_lag_1['Fatalities']
train['lag_2_ConfirmedCases']=train_lag_2['ConfirmedCases']
train['lag_2_Fatalities']=train_lag_2['Fatalities']
train['lag_3_ConfirmedCases']=train_lag_3['ConfirmedCases']
train['lag_3_Fatalities']=train_lag_3['Fatalities']
train['lag_4_ConfirmedCases']=train_lag_4['ConfirmedCases']
train['lag_4_Fatalities']=train_lag_4['Fatalities']
train
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
y = train[list_classes].values
list_sentences_train = train["comment_text"]
list_sentences_test = test["comment_text"]<string_transform>
def pred_ets(fcastperiod,fcastperiod1,actual,ffcast,type_ck='ConfirmedCases',verbose=False):
actual=actual[actual[type_ck]>0]
index=pd.date_range(start=ffcast.index[0], end=ffcast.index[-1], freq='D')
 data=ffcast[type_ck].values
ffcast1 = pd.Series(data, index)
 index=pd.date_range(start=actual.index[0], end=actual.index[-1], freq='D')
 data=actual[type_ck].values
daily_analysis_dat = pd.Series(data, index)
 livestock2=daily_analysis_dat
fit=[]
fcast=[]
fname=[]
try:
fit1 = SimpleExpSmoothing(livestock2 ).fit()
fcast1 = fit1.forecast(fcastperiod1 ).rename("SES")
 fit.append(fit1)
 fcast.append(fcast1)
 fname.append('SES')
 except:
1==1
try:
fit2 = Holt(livestock2 ).fit()
fcast2 = fit2.forecast(fcastperiod1 ).rename("Holt")
 fit.append(fit2)
 fcast.append(fcast2)
 fname.append('Holt')
 except:
1==1
try:
fit3 = Holt(livestock2, exponential=True ).fit()
fcast3 = fit3.forecast(fcastperiod1 ).rename("Exponential")
 fit.append(fit3)
 fcast.append(fcast3)
 fname.append('Exponential')
 except:
1==1
try:
fit4 = Holt(livestock2, damped=True ).fit(damping_slope=0.98)
 fcast4 = fit4.forecast(fcastperiod1 ).rename("AdditiveDamped")
 fit.append(fit4)
 fcast.append(fcast4)
 fname.append('AdditiveDamped')
 except:
1==1
try:
fit5 = Holt(livestock2, exponential=True, damped=True ).fit()
fcast5 = fit5.forecast(fcastperiod1 ).rename("MultiplicativeDamped")
 fit.append(fit5)
 fcast.append(fcast5)
 fname.append('MultiplicativeDamped')
 except:
1==1
try:
fit6 = Holt(livestock2, damped=True ).fit()
fcast6 = fit6.forecast(fcastperiod1 ).rename("AdditiveDampedC")
 fit.append(fit6)
 fcast.append(fcast6)
 fname.append('AdditiveDampedC')
 except:
1==1
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)))
 pred_all_result=pd.concat([pd.DataFrame(k.fittedvalues)for k in fit],axis=1)
 pred_all_result.columns=fname
all_result=pd.concat([pd.DataFrame(k)for k in fcast],axis=1)
 col_chk=[]
vvvl=ffcast[type_ck].values.shape[0]
for k in all_result.columns:
if verbose: print("actual value for method %s is = %s" %(k,RMSLE(all_result[k].values,ffcast[type_ck].values)))
 if RMSLE(all_result[k].values[:vvvl],ffcast[type_ck].values)is not np.nan:
col_chk.append(k)
 col_chk_f=[]
min_acc=-1
for k in col_chk:
acc=RMSLE(pred_all_result[k].values,actual[type_ck].values)
 if verbose: print("pred value for method %s is = %s" %(k,acc))
if acc is not np.nan:
col_chk_f.append(k)
 if min_acc==-1:
min_acc=acc
model_select=k
elif acc<min_acc:
min_acc=acc
model_select=k
all_result=all_result.append(pred_all_result,sort=False)
 all_result['best_model']=model_select
all_result['best_pred']=all_result[model_select]
return all_result
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(list_sentences_train))
list_tokenized_train = tokenizer.texts_to_sequences(list_sentences_train)
 list_tokenized_test = tokenizer.texts_to_sequences(list_sentences_test)
 X_t = pad_sequences(list_tokenized_train, maxlen=maxlen)
 X_te = pad_sequences(list_tokenized_test, maxlen=maxlen )<define_variables>
Fatalities_all_result_final=pd.DataFrame()
ConfirmedCases_all_result_Final=pd.DataFrame()
for keys in train['key'].unique() :
chk=train[train['key']==keys]
chk.index=chk.Date
fcastperiod=0
fcastperiod1=35
actual=chk[:chk.shape[0]-fcastperiod]
ffcast=chk[chk.shape[0]-fcastperiod-1:]
ffcast
try:
Fatalities_all_result_1=pred_ets(fcastperiod,fcastperiod1,actual,ffcast,'Fatalities' ).reset_index()
except:
Fatalities_all_result_1=pd.DataFrame(pd.date_range(start=chk.Date.min() , periods=60+fcastperiod1+1, freq='D')[1:])
 Fatalities_all_result_1.columns=['index']
Fatalities_all_result_1['best_model']='naive'
Fatalities_all_result_1['best_pred']=0
Fatalities_all_result_1['key']=keys
Fatalities_all_result_final=Fatalities_all_result_final.append(Fatalities_all_result_1,sort=True)
 try:
ConfirmedCases_all_result_1=pred_ets(fcastperiod,fcastperiod1,actual,ffcast,'ConfirmedCases' ).reset_index()
except:
ConfirmedCases_all_result_1=pd.DataFrame(pd.date_range(start=train.Date.min() , periods=60+fcastperiod1+1, freq='D')[1:])
 ConfirmedCases_all_result_1.columns=['index']
ConfirmedCases_all_result_1['best_model']='naive'
ConfirmedCases_all_result_1['best_pred']=1
ConfirmedCases_all_result_1['key']=keys
ConfirmedCases_all_result_Final=ConfirmedCases_all_result_Final.append(ConfirmedCases_all_result_1,sort=True)
 print(' done for %s' % keys)
 sys.stdout = orig_stdout
comp = 'crawl300d2m/'
embedding_path = f'{path}{comp}crawl-300d-2M.vec'<compute_test_metric>
ConfirmedCases_all_result_Final.rename(columns={'index':'Date'},inplace=True)
 Fatalities_all_result_final.rename(columns={'index':'Date'},inplace=True)
 ConfirmedCases_all_result_Final['best_pred']=np.where(ConfirmedCases_all_result_Final['best_pred'] is np.nan , 0, ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred']=np.where(Fatalities_all_result_final['best_pred'] is np.nan , 0 , Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred']=np.where(ConfirmedCases_all_result_Final['best_pred'] <0 , 0, ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred']=np.where(Fatalities_all_result_final['best_pred'] <0 , 0 , Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['AdditiveDamped'] is np.nan , ConfirmedCases_all_result_Final['best_pred'] , ConfirmedCases_all_result_Final['AdditiveDamped'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['AdditiveDamped'] is np.nan , Fatalities_all_result_final['best_pred'] , Fatalities_all_result_final['AdditiveDamped'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['best_pred'] is np.nan , 0, ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['best_pred'] is np.nan , 0 , Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['best_pred'] <0 , 0, ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['best_pred'] <0 , 0 , Fatalities_all_result_final['best_pred'])
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embedding_index = dict(get_coefs(*o.strip().split(" ")) for o in open(embedding_path))<feature_engineering>
test['Date']=test.Date.astype('datetime64[ns]' )
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embedding_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<import_modules>
eval1 = ConfirmedCases_all_result_Final[['key','Date','best_pred','best_pred_1']].merge(test, how='right', on=['key','Date'])
 eval1.rename(columns={'best_pred':'ConfirmedCases'},inplace=True)
 eval1['ConfirmedCases']=eval1['ConfirmedCases'].fillna(0)
 eval1
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Flatten, Dense, Dropout, Activation, LSTM, Bidirectional, GlobalMaxPool1D, Embedding,AveragePooling1D
from tensorflow.keras import Input, Model
from tensorflow.keras import layers, initializers, regularizers, constraints, optimizers
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.callbacks import ModelCheckpoint
from tensorflow.compat.v1.keras.layers import CuDNNGRU, CuDNNLSTM<import_modules>
eval2 = Fatalities_all_result_final[['key','Date','best_pred','best_pred_1']].merge(test, how='right', on=['key','Date'])
 eval2.rename(columns={'best_pred':'Fatalities'},inplace=True)
 eval2['Fatalities']=eval2['Fatalities'].fillna(0)
 eval2
from tensorflow.keras.layers import LeakyReLU
from keras.layers import GRU<prepare_x_and_y>
sub_prep = eval1[['ForecastId','ConfirmedCases','key']].merge(eval2[['ForecastId','Fatalities']], on=['ForecastId'], how='left')
 sub_prep
x=0
model = 0<choose_model_class>
sub = sub_prep.merge(submission['ForecastId'], on=['ForecastId'], how='right')
 sub
maxlen=200
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x= SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Conv1D(64, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform" )(x)
 avg_pool = GlobalAvgPool1D()(x)
 max_pool = GlobalMaxPool1D()(x)
 conc = concatenate([avg_pool,max_pool])
 x = Dense(50 )(conc)
 x = LeakyReLU(alpha = 0.01 )(x)
 x = Dropout(0.25 )(x)
 x = Dense(6, activation = "sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.summary()<choose_model_class>
sub=sub[['ForecastId','ConfirmedCases','Fatalities']]
sub=sub.sort_values('ForecastId')
 sub
model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'] )<choose_model_class>
sub.to_csv('submission.csv',header=['ForecastId','ConfirmedCases','Fatalities'],index=False)
 sub
file_path = "bi_gru.best.hdf5"
checkpoint = ModelCheckpoint(file_path, monitor='val_loss',verbose=1, save_best_only=True, mode = 'min')
 early = EarlyStopping(monitor="val_loss", mode="min",patience=1 )<train_model>
train['Date']=train.Date.astype('datetime64[ns]')
 verify=train[['key','Date','ConfirmedCases','Fatalities']].merge(test[['key','Date','ForecastId']], how='inner', on=['key','Date'])
 pred=verify[['ForecastId']].merge(sub, how='inner', on=['ForecastId'] )
batch_size = 32
epochs = 10
callbacks_list = [checkpoint,early]
history = model.fit(X_t,y, batch_size=batch_size, epochs=epochs, validation_split=0.1, callbacks=callbacks_list )<predict_on_test>
RMSLE(pred['Fatalities'].values,verify['Fatalities'].values )
batch_size = 1024
results = model.predict(X_te, batch_size=batch_size,verbose = 1 )<save_to_csv>
RMSLE(pred['ConfirmedCases'].values,verify['ConfirmedCases'].values )
sample_submission[list_classes] = results
sample_submission.to_csv('fin.csv', index=False)
 files.download('fin.csv' )<set_options>
train
train_ck=train.groupby(['Country/Region','Date'] ).sum().reset_index()
train_ck['key']=train_ck['Country/Region']
%matplotlib inline
%matplotlib inline 
<define_variables>
Fatalities_all_result_final=pd.DataFrame()
ConfirmedCases_all_result_Final=pd.DataFrame()
for keys in train_ck['key'].unique() :
chk=train_ck[train_ck['key']==keys]
chk.index=chk.Date
fcastperiod=0
fcastperiod1=35
actual=chk[:chk.shape[0]-fcastperiod]
ffcast=chk[chk.shape[0]-fcastperiod-1:]
ffcast
try:
Fatalities_all_result_1=pred_ets(fcastperiod,fcastperiod1,actual,ffcast,'Fatalities' ).reset_index()
except:
Fatalities_all_result_1=pd.DataFrame(pd.date_range(start=train.Date.min() , periods=60+fcastperiod1+1, freq='D')[1:])
 Fatalities_all_result_1.columns=['index']
Fatalities_all_result_1['best_model']='naive'
Fatalities_all_result_1['best_pred']=0
Fatalities_all_result_1['key']=keys
Fatalities_all_result_final=Fatalities_all_result_final.append(Fatalities_all_result_1,sort=True)
 try:
ConfirmedCases_all_result_1=pred_ets(fcastperiod,fcastperiod1,actual,ffcast,'ConfirmedCases' ).reset_index()
except:
ConfirmedCases_all_result_1=pd.DataFrame(pd.date_range(start=train.Date.min() , periods=60+fcastperiod1+1, freq='D')[1:])
 ConfirmedCases_all_result_1.columns=['index']
ConfirmedCases_all_result_1['best_model']='naive'
ConfirmedCases_all_result_1['best_pred']=1
ConfirmedCases_all_result_1['key']=keys
ConfirmedCases_all_result_Final=ConfirmedCases_all_result_Final.append(ConfirmedCases_all_result_1,sort=True)
 print(' done for %s' % keys )
path = '.. /input/'
comp = 'jigsaw-toxic-comment-classification-challenge/'
EMBEDDING_FILE=f'{path}glove6b50d/glove.6B.50d.txt'
TRAIN_DATA_FILE=f'{path}{comp}train.csv.zip'
TEST_DATA_FILE=f'{path}{comp}test.csv.zip'
TEST_LABELS_FILE = f'{path}{comp}test_labels.csv.zip'<define_variables>
ConfirmedCases_all_result_Final.rename(columns={'index':'Date'},inplace=True)
 Fatalities_all_result_final.rename(columns={'index':'Date'},inplace=True)
 ConfirmedCases_all_result_Final['best_pred']=np.where(ConfirmedCases_all_result_Final['best_pred'] is np.nan , 0, ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred']=np.where(Fatalities_all_result_final['best_pred'] is np.nan , 0 , Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred']=np.where(ConfirmedCases_all_result_Final['best_pred'] <0 , 0, ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred']=np.where(Fatalities_all_result_final['best_pred'] <0 , 0 , Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['AdditiveDamped'] is np.nan , ConfirmedCases_all_result_Final['best_pred'] , ConfirmedCases_all_result_Final['AdditiveDamped'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['AdditiveDamped'] is np.nan , Fatalities_all_result_final['best_pred'] , Fatalities_all_result_final['AdditiveDamped'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['best_pred'] is np.nan , 0, ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['best_pred'] is np.nan , 0 , Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['best_pred'] <0 , 0, ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['best_pred'] <0 , 0 , Fatalities_all_result_final['best_pred'] )
embed_size = 50
max_features = 20000
maxlen = 200<load_from_csv>
best_model_key=ConfirmedCases_all_result_Final[['key','best_model']].drop_duplicates()
max_number_current=train_ck.groupby('key' ).max() [['ConfirmedCases','Fatalities']].reset_index()
best_model_key=best_model_key.merge(max_number_current,on='key',how='left')
 best_model_key=best_model_key.sort_values('ConfirmedCases',ascending=False ).reset_index(drop=True )