Kaggle competition: Natural Language Processing with Disaster Tweets
train2_x = features[['Lat','Long','Date']]
train2_y = features[['Fatalities']]
test2_x = test_features[['Lat','Long','Date']]<predict_on_test>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import re
import string
from tqdm import tqdm
from gensim.parsing.preprocessing import remove_stopwords
from bs4 import BeautifulSoup
from nltk.stem.snowball import SnowballStemmer
from nltk.stem.wordnet import WordNetLemmatizer
from collections import OrderedDict
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split,RandomizedSearchCV
from sklearn.metrics import classification_report,f1_score
import tensorflow as tf
import tensorflow_hub as hub
from tensorflow import keras
from keras import backend as K
from tensorflow.keras.preprocessing import sequence
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.optimizers import Adam
import torch
import transformers
model = RandomForestClassifier(max_depth=200, random_state=1)
 model.fit(train2_x, train2_y)
 fatalities = model.predict(test2_x )<load_from_csv>
train = pd.read_csv(r'/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv(r'/kaggle/input/nlp-getting-started/test.csv' )
Sub = pd.read_csv('.. /input/covid19-global-forecasting-week-1/submission.csv')
 output = pd.DataFrame({
'ForecastId': Sub['ForecastId'],
'ConfirmedCases': conf_cases,
'Fatalities': fatalities
} )<save_to_csv>
def remove_shortforms(phrase):
phrase = re.sub(r"won't", "will not", phrase)
 phrase = re.sub(r"can't", "can not", phrase)
 phrase = re.sub(r"n't", " not", phrase)
 phrase = re.sub(r"'re", " are", phrase)
 phrase = re.sub(r"'s", " is", phrase)
 phrase = re.sub(r"'d", " would", phrase)
 phrase = re.sub(r"'ll", " will", phrase)
 phrase = re.sub(r"'t", " not", phrase)
 phrase = re.sub(r"'ve", " have", phrase)
 phrase = re.sub(r"'m", " am", phrase)
 return phrase
def remove_special_char(text):
text = re.sub('[^A-Za-z0-9]+'," ",text)
 return text
def remove_wordswithnum(text):
text = re.sub("\S*\d\S*", "", text ).strip()
return text
def lowercase(text):
text = text.lower()
return text
def remove_stop_words(text):
text = remove_stopwords(text)
 return text
st = SnowballStemmer(language='english')
 def stemming(text):
r= []
for word in text :
a = st.stem(word)
 r.append(a)
 return r
def listToString(s):
str1 = " "
return(str1.join(s))
def remove_punctuations(text):
text = re.sub(r'[^\w\s]','',text)
 return text
def remove_links(text):
text = re.sub(r'http\S+', '', text)
 return text
lemmatizer = WordNetLemmatizer()
def lemmatize_words(text):
text = lemmatizer.lemmatize(text)
 return text
def remove_html(text):
html=re.compile(r'<.*?>')
 return html.sub(r'',text )
output.to_csv('submission.csv', index=False )<import_modules>
Y = train['target']
train = train.drop('target',axis=1)
 data = pd.concat([train,test],axis=0 ).reset_index(drop=True)
 data.head()
import numpy as np
import pandas as pd
import xgboost as xgb
from xgboost import plot_importance, plot_tree
from sklearn.metrics import mean_squared_error, mean_absolute_error
from sklearn.ensemble import RandomForestRegressor<load_from_csv>
for i in range(len(data['text'])) :
data['text'][i] = str(data['text'][i] )
train = pd.read_csv(".. /input/covid19-global-forecasting-week-1/train.csv")
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-1/test.csv" )<drop_column>
for i in range(len(data['text'])) :
data['text'][i] = remove_shortforms(data['text'][i])
 data['text'][i] = remove_special_char(data['text'][i])
 data['text'][i] = remove_wordswithnum(data['text'][i])
 data['text'][i] = lowercase(data['text'][i])
 data['text'][i] = remove_stop_words(data['text'][i])
 text = data['text'][i]
text = text.split()
data['text'][i] = stemming(text)
 s = data['text'][i]
data['text'][i] = listToString(s)
 data['text'][i] = lemmatize_words(data['text'][i] )
X_train = train.drop(["Fatalities", "ConfirmedCases"], axis=1 )<define_variables>
cv = CountVectorizer(ngram_range=(1,3))
text_bow = cv.fit_transform(data['text'])
 print(text_bow.shape )
countries = X_train["Country/Region"]<drop_column>
train_text = text_bow[:train.shape[0]]
test_text = text_bow[train.shape[0]:]
X_train = X_train.drop(["Id"], axis=1)
 X_test = test.drop(["ForecastId"], axis=1 )<data_type_conversions>
X_train,X_test,Y_train,Y_test = train_test_split(train_text,Y,test_size=0.2)
 print(X_train.shape)
 print(X_test.shape)
 print(Y_train.shape)
 print(Y_test.shape )
X_train['Date']= pd.to_datetime(X_train['Date'])
 X_test['Date']= pd.to_datetime(X_test['Date'] )<rename_columns>
lr = LogisticRegression(C=10,penalty='l2')
 lr.fit(X_train,Y_train)
 pred = lr.predict(X_test)
 print("F1 score :",f1_score(Y_test,pred))
print("Classification Report
:",classification_report(Y_test,pred))
X_train = X_train.set_index(['Date'])
 X_test = X_test.set_index(['Date'] )<feature_engineering>
lr = LogisticRegression(C=10,penalty='l2',max_iter=2000)
 lr.fit(train_text,Y)
 pred = lr.predict(test_text)
 submit = pd.DataFrame(test['id'],columns=['id'])
 print(len(pred))
submit.head()
def create_time_features(df):

 df['date'] = df.index
df['hour'] = df['date'].dt.hour
df['dayofweek'] = df['date'].dt.dayofweek
df['quarter'] = df['date'].dt.quarter
df['month'] = df['date'].dt.month
df['year'] = df['date'].dt.year
df['dayofyear'] = df['date'].dt.dayofyear
df['dayofmonth'] = df['date'].dt.day
df['weekofyear'] = df['date'].dt.weekofyear
X = df[['hour','dayofweek','quarter','month','year',
'dayofyear','dayofmonth','weekofyear']]
return X<feature_engineering>
submit['target'] = pred
submit.to_csv("realnlp.csv",index=False )
create_time_features(X_train)
 create_time_features(X_test )<drop_column>
tfidf = TfidfVectorizer(ngram_range=(1,3))
text_tfidf = tfidf.fit_transform(data['text'])
 print(text_tfidf.shape )
X_train.drop("date", axis=1, inplace=True)
 X_test.drop("date", axis=1, inplace=True )<categorify>
X_train,X_test,Y_train,Y_test = train_test_split(train_text,Y,test_size=0.2)
 print(X_train.shape)
 print(X_test.shape)
 print(Y_train.shape)
 print(Y_test.shape )
X_train = pd.concat([X_train,pd.get_dummies(X_train['Province/State'], prefix='ps')],axis=1)
 X_train.drop(['Province/State'],axis=1, inplace=True)
 X_test = pd.concat([X_test,pd.get_dummies(X_test['Province/State'], prefix='ps')],axis=1)
 X_test.drop(['Province/State'],axis=1, inplace=True )<categorify>
lr = LogisticRegression(C=100,penalty='l2',max_iter=2000)
 lr.fit(X_train,Y_train)
 pred = lr.predict(X_test)
 print("F1 score :",f1_score(Y_test,pred))
print("Classification Report :",classification_report(Y_test,pred))
X_train = pd.concat([X_train,pd.get_dummies(X_train['Country/Region'], prefix='cr')],axis=1)
 X_train.drop(['Country/Region'],axis=1, inplace=True)
 X_test = pd.concat([X_test,pd.get_dummies(X_test['Country/Region'], prefix='cr')],axis=1)
 X_test.drop(['Country/Region'],axis=1, inplace=True )<prepare_x_and_y>
print("Number of null values in data keywords column : ",data['keyword'].isnull().sum() )
y_train = train["Fatalities"]<load_from_csv>
data['keyword'] = data['keyword'].fillna("unknown")
 data.head()
submissionOrig = pd.read_csv(".. /input/covid19-global-forecasting-week-1/submission.csv" )<predict_on_test>
combined_text = [None] * len(data['text'])
 for i in range(len(data['text'])) :
if data['keyword'][i] == 'unknown':
combined_text[i] = data['text'][i]
else:
combined_text[i] = data['text'][i] + " " + data['keyword'][i] + " " + data['keyword'][i] + " " + data['keyword'][i]
data['combined_text'] = combined_text
y_train = train["ConfirmedCases"]
confirmed_reg = RandomForestRegressor(max_depth=100,n_jobs=-1,n_estimators=100)
 confirmed_reg.fit(X_train, y_train)
 preds = confirmed_reg.predict(X_test)
 preds = np.array(preds)
 preds[preds < 0] = 0
preds = np.round(preds, 0)
 preds = np.array(preds)
 submissionOrig["ConfirmedCases"]=pd.Series(preds )<predict_on_test>
for i in range(len(data['combined_text'])) :
data['combined_text'][i] = str(data['combined_text'][i] )
y_train = train["Fatalities"]
confirmed_reg = RandomForestRegressor(max_depth=100,n_jobs=-1,n_estimators=100)
 confirmed_reg.fit(X_train, y_train)
 preds = confirmed_reg.predict(X_test)
 preds = np.array(preds)
 preds[preds < 0] = 0
preds = np.round(preds, 0)
 submissionOrig["Fatalities"]=pd.Series(preds)
<save_to_csv>
for i in range(len(data['combined_text'])) :
data['combined_text'][i] = remove_shortforms(data['combined_text'][i])
 data['combined_text'][i] = remove_special_char(data['combined_text'][i])
 data['combined_text'][i] = remove_wordswithnum(data['combined_text'][i])
 data['combined_text'][i] = lowercase(data['combined_text'][i])
 data['combined_text'][i] = remove_stop_words(data['combined_text'][i])
 text = data['combined_text'][i]
text = text.split()
data['combined_text'][i] = stemming(text)
 s = data['combined_text'][i]
data['combined_text'][i] = listToString(s)
 data['combined_text'][i] = lemmatize_words(data['combined_text'][i] )
submissionOrig.to_csv('submission.csv',index=False )<load_from_csv>
cv = CountVectorizer(ngram_range=(1,3))
text_bow = cv.fit_transform(data['combined_text'])
 print(text_bow.shape )
path = '.. /input/covid19-global-forecasting-week-1/'
train = pd.read_csv(path + 'train.csv')
 test = pd.read_csv(path + 'test.csv')
 sub = pd.read_csv(path + 'submission.csv')
 train['Date'] = train['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 test['Date'] = test['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 path_ext = '.. /input/novel-corona-virus-2019-dataset/'
ext_rec = pd.read_csv(path_ext + 'time_series_covid_19_recovered.csv' ).\
melt(id_vars=["Province/State", "Country/Region", "Lat", "Long"],
var_name="Date",
value_name="Recoveries")
 ext_rec['Date'] = ext_rec['Date'].apply(lambda x:(datetime.datetime.strptime(x+"20", '%m/%d/%Y')))
 train = train.merge(ext_rec[['Province/State', 'Country/Region', 'Date', 'Recoveries']], how='left',
left_on=['Province/State', 'Country/Region', 'Date'],
right_on=['Province/State', 'Country/Region', 'Date'])
 train['days'] =(train['Date'].dt.date - train['Date'].dt.date.min() ).dt.days
test['days'] =(test['Date'].dt.date - train['Date'].dt.date.min() ).dt.days
train.loc[train['Province/State'].isnull() , 'Province/State'] = 'N/A'
test.loc[test['Province/State'].isnull() , 'Province/State'] = 'N/A'
train['Area'] = train['Country/Region'] + '_' + train['Province/State']
test['Area'] = test['Country/Region'] + '_' + test['Province/State']
print(train['Date'].max())
 print(train['days'].max())
 N_AREAS = train['Area'].nunique()
AREAS = np.sort(train['Area'].unique())
 TRAIN_N = 50 + 13
print(train[train['days'] < TRAIN_N]['Date'].max())
 train.head()<prepare_x_and_y>
train_text = text_bow[:train.shape[0]]
test_text = text_bow[train.shape[0]:]
train_p_c = train.pivot(index='Area', columns='days', values='ConfirmedCases' ).sort_index()
train_p_f = train.pivot(index='Area', columns='days', values='Fatalities' ).sort_index()
train_p_c = np.maximum.accumulate(train_p_c, axis=1)
 train_p_f = np.maximum.accumulate(train_p_f, axis=1)
 f_rate =(train_p_f / train_p_c ).fillna(0)
 X_c = np.log(1+train_p_c.values)
 X_f = train_p_f.values 
<compute_train_metric>
X_train,X_test,Y_train,Y_test = train_test_split(train_text,Y,test_size=0.2)
 print(X_train.shape)
 print(X_test.shape)
 print(Y_train.shape)
 print(Y_test.shape )
def eval1(y, p):
val_len = y.shape[1] - TRAIN_N
return np.sqrt(mean_squared_error(y[:, TRAIN_N:TRAIN_N+val_len].flatten() , p[:, TRAIN_N:TRAIN_N+val_len].flatten()))
def run_c(params, X, test_size=50):
gr_base = []
for i in range(X_c.shape[0]):
temp = X[i,:]
threshold = np.log(1+params['min cases for growth rate'])
 num_days = params['last N days']
if(temp > threshold ).sum() > num_days:
gr_base.append(np.clip(np.diff(temp[temp > threshold])[-num_days:].mean() , 0, params['growth rate max']))
else:
gr_base.append(params['growth rate default'])
 gr_base = np.array(gr_base)
 preds = X.copy()
for i in range(test_size):
delta = np.clip(preds[:, -1], np.log(2), None)+ gr_base *(1 + params['growth rate factor'])**i
preds = np.hstack(( preds, delta.reshape(-1,1)))
 return preds
params = {
"min cases for growth rate": 30,
"last N days": 8,
"growth rate default": 0.20,
"growth rate max": 0.3,
"growth rate factor": -0.09,
}
preds_c = run_c(params, np.log(1+train_p_c.values)[:,:TRAIN_N])
<train_model>
lr = LogisticRegression(C=1,penalty='l2',max_iter=2000)
 lr.fit(X_train,Y_train)
 pred = lr.predict(X_test)
 print("F1 score :",f1_score(Y_test,pred))
print("Classification Report :",classification_report(Y_test,pred))
def run_f(params, X_c, X_f, X_f_r, test_size=50):
X_f_r = np.array(np.ma.mean(np.ma.masked_outside(X_f_r, 0.06, 0.4)[:,:], axis=1))
X_f_r = np.clip(X_f_r, params['fatality_rate_lower'], params['fatality_rate_upper'])
 X_c = np.clip(np.exp(X_c)-1, 0, None)
 preds = X_f.copy()
train_size = X_f.shape[1] - 1
for i in range(test_size):
delta = X_c[:, train_size+i-params['length']+1] - X_c[:, train_size+i-params['length']]
delta = params['absolute growth'] + delta*X_f_r
preds = np.hstack(( preds, preds[:, -1].reshape(-1,1)+ delta.reshape(-1,1)))
 return preds
params = {
"length": 8,
"absolute growth": 0.02,
"fatality_rate_lower": 0.035,
"fatality_rate_upper": 0.40,
}
preds_f = run_f(params, preds_c, X_f, f_rate.values[:,:TRAIN_N])
 preds_f = np.log(1+preds_f)
<compute_test_metric>
tfidf = TfidfVectorizer(ngram_range=(1,3))
text_tfidf = tfidf.fit_transform(data['combined_text'])
 print(text_tfidf.shape )
if False:
val_len = train_p_c.values.shape[1] - TRAIN_N
for i in range(val_len):
d = i + TRAIN_N
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c.values[:, d]), preds_c[:, d]))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f.values[:, d]), preds_f[:, d]))
print(f"{d}: {(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]")
 print()
m1 = np.sqrt(mean_squared_error(np.log(1 + train_p_c.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_c[:, TRAIN_N:TRAIN_N+val_len].flatten()))
m2 = np.sqrt(mean_squared_error(np.log(1 + train_p_f.values[:, TRAIN_N:TRAIN_N+val_len] ).flatten() , preds_f[:, TRAIN_N:TRAIN_N+val_len].flatten()))
print(f"{(m1 + m2)/2:8.5f} [{m1:8.5f} {m2:8.5f}]" )<categorify>
train_text = text_tfidf[:train.shape[0]]
test_text = text_tfidf[train.shape[0]:]
temp = pd.DataFrame(np.clip(np.exp(preds_c)- 1, 0, None))
temp['Area'] = AREAS
temp = temp.melt(id_vars='Area', var_name='days', value_name="ConfirmedCases")
 test = test.merge(temp, how='left', left_on=['Area', 'days'], right_on=['Area', 'days'])
 temp = pd.DataFrame(np.clip(np.exp(preds_f)- 1, 0, None))
temp['Area'] = AREAS
temp = temp.melt(id_vars='Area', var_name='days', value_name="Fatalities")
 test = test.merge(temp, how='left', left_on=['Area', 'days'], right_on=['Area', 'days'])
 test.head()<save_to_csv>
X_train,X_test,Y_train,Y_test = train_test_split(train_text,Y,test_size=0.2)
 print(X_train.shape)
 print(X_test.shape)
 print(Y_train.shape)
 print(Y_test.shape )
test.to_csv("submission.csv", index=False, columns=["ForecastId", "ConfirmedCases", "Fatalities"] )<sort_values>
lr = LogisticRegression(C=2,penalty='l2',max_iter=2000)
 lr.fit(X_train,Y_train)
 pred = lr.predict(X_test)
 print("F1 score :",f1_score(Y_test,pred))
print("Classification Report :",classification_report(Y_test,pred))
for i, rec in test.groupby('Area' ).last().sort_values("ConfirmedCases", ascending=False ).iterrows() :
print(f"{rec['ConfirmedCases']:10.1f} {rec['Fatalities']:10.1f} {rec['Country/Region']}, {rec['Province/State']}")
<set_options>
print('Loading word vectors...')
 word2vec = {}
with open(os.path.join('.. /input/glove-global-vectors-for-word-representation/glove.6B.200d.txt'), encoding = "utf-8")as f:
for line in f:
values = line.split()
word = values[0]
vec = np.asarray(values[1:], dtype='float32')
 word2vec[word] = vec
print('Found %s word vectors.' % len(word2vec))
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"
pd.set_option('display.max_columns', 99)
 pd.set_option('display.max_rows', 99)
<set_options>
train = pd.read_csv(r'/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv(r'/kaggle/input/nlp-getting-started/test.csv' )
plt.rcParams['figure.figsize'] = [16, 10]
plt.rcParams['font.size'] = 14
sns.set_palette(sns.color_palette('tab20', 20))<define_variables>
Y = train['target']
train = train.drop('target',axis=1)
 data = pd.concat([train,test],axis=0 ).reset_index(drop=True)
 text_data = data['text']
DATEFORMAT = '%Y-%m-%d'<define_variables>
tokenizer = Tokenizer()
tokenizer.fit_on_texts(text_data)
 sequences = tokenizer.texts_to_sequences(text_data )
COMP = 'covid19-global-forecasting-week-1'<load_from_csv>
word2index = tokenizer.word_index
print("Number of unique tokens : ",len(word2index))
train = pd.read_csv(f'.. /input/{COMP}/train.csv')
 test = pd.read_csv(f'.. /input/{COMP}/test.csv')
 submission = pd.read_csv(f'.. /input/{COMP}/submission.csv')
 train.shape, test.shape, submission.shape<feature_engineering>
train_pad = data_padded[:train.shape[0]]
test_pad = data_padded[train.shape[0]:]
def to_log(x):
return np.log(x + 1)
 def to_exp(x):
return np.exp(x)- 1<prepare_output>
embedding_matrix = np.zeros(( len(word2index)+1,200))
embedding_vec=[]
for word, i in tqdm(word2index.items()):
embedding_vec = word2vec.get(word)
 if embedding_vec is not None:
embedding_matrix[i] = embedding_vec
np.arange(10)
 to_exp(to_log(np.arange(10)) )<feature_engineering>
model1 = keras.models.Sequential([
keras.layers.Embedding(len(word2index)+1,200,weights=[embedding_matrix],input_length=100,trainable=False),
keras.layers.LSTM(100,return_sequences=True),
keras.layers.LSTM(200),
keras.layers.Dropout(0.5),
keras.layers.Dense(1,activation='sigmoid')
 ] )
train['Location'] = train['Country/Region'] + '-' + train['Province/State'].fillna('')
 train['Location'] = train['Location'].str.replace(',', '')
 test['Location'] = test['Country/Region'] + '-' + test['Province/State'].fillna('')
 test['Location'] = test['Location'].str.replace(',', '')
 train['LogConfirmed'] = to_log(train.ConfirmedCases)
 train['LogFatalities'] = to_log(train.Fatalities)
 train = train.drop(columns=['Province/State', 'Country/Region', 'Lat', 'Long'])
 test = test.drop(columns=['Province/State', 'Country/Region', 'Lat', 'Long'] )<feature_engineering>
model1.compile(
 loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'],
)
dfs = []
for loc, df in tqdm(train.groupby('Location')) :
df = df.sort_values(by='Date')
 df['LogFatalities'] = df['LogFatalities'].cummax()
df['LogConfirmed'] = df['LogConfirmed'].cummax()
df['LogConfirmedNextDay'] = df['LogConfirmed'].shift(-1)
 df['DateNextDay'] = df['Date'].shift(-1)
 df['LogFatalitiesNextDay'] = df['LogFatalities'].shift(-1)
 df['LogConfirmedDelta'] = df['LogConfirmedNextDay'] - df['LogConfirmed']
df['LogFatalitiesDelta'] = df['LogFatalitiesNextDay'] - df['LogFatalities']
dfs.append(df)
 dfs = pd.concat(dfs )<save_to_csv>
history1 = model1.fit(train_pad,Y,
batch_size=64,
epochs=10,
validation_split=0.2
)
deltas = dfs[np.logical_and(
 dfs.LogConfirmed > 0,
~dfs.Location.str.startswith('China')
)].dropna().sort_values(by='LogConfirmedDelta', ascending=False)
 deltas['start'] = deltas['LogConfirmed'].round(1)
 confirmed_deltas = pd.concat([
deltas.groupby('start')[['LogConfirmedDelta']].mean() ,
deltas.groupby('start')[['LogConfirmedDelta']].std() ,
deltas.groupby('start')[['LogConfirmedDelta']].count()
], axis=1)
 deltas.mean()
confirmed_deltas.columns = ['avg', 'std', 'cnt']
confirmed_deltas
confirmed_deltas.to_csv('confirmed_deltas.csv' )<define_variables>
model2 = keras.models.Sequential([
keras.layers.Embedding(len(word2index)+1,200,weights=[embedding_matrix],input_length=100,trainable=False),
keras.layers.GRU(100,return_sequences=True),
keras.layers.GRU(200),
keras.layers.Dropout(0.5),
keras.layers.Dense(1,activation='sigmoid')
 ] )
DECAY = 0.93
DECAY ** 14, DECAY ** 27<feature_engineering>
model2.compile(
 loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'],
)
confirmed_deltas = train.groupby('Location')[['Id']].count()
confirmed_deltas['DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.index.str.startswith('China'), 'DELTA'] = 0.02
confirmed_deltas.loc[confirmed_deltas.index.str.startswith('US'), 'DELTA'] = 0.24
confirmed_deltas.loc[confirmed_deltas.index=='Turkey-', 'DELTA'] = 0.23
confirmed_deltas.loc[confirmed_deltas.index=='Brazil-', 'DELTA'] = 0.23
confirmed_deltas.loc[confirmed_deltas.index=='Mexico-', 'DELTA'] = 0.23
confirmed_deltas.loc[confirmed_deltas.index=='Romania-', 'DELTA'] = 0.25
confirmed_deltas.loc[confirmed_deltas.index=='Singapore-', 'DELTA'] = 0.07
confirmed_deltas.loc[confirmed_deltas.index=='Japan-', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.index=='Korea South-', 'DELTA'] = 0.03
confirmed_deltas.loc[confirmed_deltas.index=='China-Hong Kong', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.index=='Iran-', 'DELTA'] = 0.14
confirmed_deltas.loc[confirmed_deltas.index=='Italy-', 'DELTA'] = 0.14
confirmed_deltas.loc[confirmed_deltas.index.str.endswith('Princess'), 'DELTA'] = 0.01
confirmed_deltas<save_to_csv>
history2 = model2.fit(train_pad,Y,
batch_size=64,
epochs=10,
validation_split=0.2
)
daily_log_confirmed = dfs.pivot('Location', 'Date', 'LogConfirmed' ).reset_index()
daily_log_confirmed = daily_log_confirmed.sort_values('2020-03-24', ascending=False)
 daily_log_confirmed.to_csv('daily_log_confirmed.csv', index=False)
 for i, d in tqdm(enumerate(pd.date_range('2020-03-25', '2020-04-24'))):
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in confirmed_deltas.index:
confirmed_delta = confirmed_deltas.loc[confirmed_deltas.index == loc, 'DELTA'].values[0]
daily_log_confirmed.loc[daily_log_confirmed.Location == loc, new_day] = daily_log_confirmed.loc[daily_log_confirmed.Location == loc, last_day] + \
confirmed_delta * DECAY ** i<feature_engineering>
model3 = keras.models.Sequential([
keras.layers.Embedding(len(word2index)+1,200,weights=[embedding_matrix],input_length=100,trainable=False),
keras.layers.Bidirectional(keras.layers.LSTM(100,return_sequences=True)) ,
keras.layers.Bidirectional(keras.layers.LSTM(200)) ,
keras.layers.Dropout(0.5),
keras.layers.Dense(1,activation='sigmoid')
 ] )
confirmed_deltas = train.groupby('Location')[['Id']].count()
confirmed_deltas['DELTA'] = 0.18
confirmed_deltas.loc[confirmed_deltas.index.str.startswith('China'), 'DELTA'] = 0.02
confirmed_deltas.loc[confirmed_deltas.index.str.startswith('US'), 'DELTA'] = 0.2
confirmed_deltas.loc[confirmed_deltas.index=='Turkey-', 'DELTA'] = 0.23
confirmed_deltas.loc[confirmed_deltas.index=='Brazil-', 'DELTA'] = 0.23
confirmed_deltas.loc[confirmed_deltas.index=='Mexico-', 'DELTA'] = 0.23
confirmed_deltas.loc[confirmed_deltas.index=='Romania-', 'DELTA'] = 0.25
confirmed_deltas.loc[confirmed_deltas.index=='Singapore-', 'DELTA'] = 0.07
confirmed_deltas.loc[confirmed_deltas.index=='Japan-', 'DELTA'] = 0.05
confirmed_deltas.loc[confirmed_deltas.index=='Korea South-', 'DELTA'] = 0.03
confirmed_deltas.loc[confirmed_deltas.index=='China-Hong Kong', 'DELTA'] = 0.12
confirmed_deltas.loc[confirmed_deltas.index=='Iran-', 'DELTA'] = 0.14
confirmed_deltas.loc[confirmed_deltas.index=='Italy-', 'DELTA'] = 0.14
confirmed_deltas.loc[confirmed_deltas.index.str.endswith('Princess'), 'DELTA'] = 0.01
confirmed_deltas
confirmed_deltas.describe()<sort_values>
model3.compile(
 loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'],
)
death_deltas = dfs[np.logical_and(
 dfs.Fatalities > 0,
~dfs.Location.str.startswith('China')
)].dropna().sort_values(by='LogFatalitiesDelta', ascending=False )<save_to_csv>
history3 = model3.fit(train_pad,Y,
batch_size=64,
epochs=10,
validation_split=0.2
)
loc_confirmed_death_deltas = pd.concat([
death_deltas.groupby('Location')[['LogFatalitiesDelta']].mean() ,
death_deltas.groupby('Location')[['LogFatalitiesDelta']].std() ,
death_deltas.groupby('Location')[['LogFatalitiesDelta']].count() ,
death_deltas.groupby('Location')[['LogFatalities']].max()
], axis=1)
 loc_confirmed_death_deltas.columns = ['avg', 'std', 'cnt', 'max']
loc_confirmed_death_deltas = loc_confirmed_death_deltas[loc_confirmed_death_deltas.cnt >= 3]
loc_confirmed_death_deltas.sort_values(by='avg' ).head(10)
 loc_confirmed_death_deltas.sort_values(by='avg' ).tail(10)
 loc_confirmed_death_deltas.to_csv('loc_confirmed_death_deltas.csv' )<feature_engineering>
es = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy',mode='max',verbose=1,patience=3 )
death_deltas = train.groupby('Location')[['Id']].count()
death_deltas['DELTA'] = 0.17
death_deltas.loc[death_deltas.index.str.startswith('China'), 'DELTA'] = 0.01
death_deltas.loc[death_deltas.index.str.startswith('US'), 'DELTA'] = 0.15
death_deltas.loc[death_deltas.index=='Turkey-', 'DELTA'] = 0.23
death_deltas.loc[death_deltas.index=='Brazil-', 'DELTA'] = 0.23
death_deltas.loc[death_deltas.index=='Mexico-', 'DELTA'] = 0.23
death_deltas.loc[death_deltas.index=='Singapore-', 'DELTA'] = 0.07
death_deltas.loc[death_deltas.index=='Japan-', 'DELTA'] = 0.05
death_deltas.loc[death_deltas.index=='Korea South-', 'DELTA'] = 0.05
death_deltas.loc[death_deltas.index=='Taiwan*-', 'DELTA'] = 0.05
death_deltas.loc[death_deltas.index=='China-Hong Kong', 'DELTA'] = 0.08
death_deltas.loc[death_deltas.index=='Iran-', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.index=='Italy-', 'DELTA'] = 0.12
death_deltas.loc[death_deltas.index=='Spain-', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.index.str.endswith('Princess'), 'DELTA'] = 0.01
death_deltas<save_to_csv>
history = model3.fit(train_pad,Y,
batch_size=64,
epochs=30,
validation_split=0.2,
callbacks=[es]
)
daily_log_deaths = dfs.pivot('Location', 'Date', 'LogFatalities' ).reset_index()
daily_log_deaths = daily_log_deaths.sort_values('2020-03-24', ascending=False)
 daily_log_deaths.to_csv('daily_log_deaths.csv', index=False)
 for i, d in tqdm(enumerate(pd.date_range('2020-03-25', '2020-04-24'))):
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in death_deltas.index:
death_delta = death_deltas.loc[death_deltas.index == loc, 'DELTA'].values[0]
daily_log_deaths.loc[daily_log_deaths.Location == loc, new_day] = daily_log_deaths.loc[daily_log_deaths.Location == loc, last_day] + \
death_delta * DECAY ** i<define_variables>
submit = pd.DataFrame(test['id'],columns=['id'])
 predictions = model3.predict(test_pad)
 submit['target_prob'] = predictions
submit.head()
confirmed = []
fatalities = []
for id, d, loc in tqdm(test.values):
c = to_exp(daily_log_confirmed.loc[daily_log_confirmed.Location == loc, d].values[0])
 f = to_exp(daily_log_deaths.loc[daily_log_deaths.Location == loc, d].values[0])
 confirmed.append(c)
 fatalities.append(f )<prepare_output>
target = [None]*len(submit)
 for i in range(len(submit)) :
target[i] = np.round(submit['target_prob'][i] ).astype(int)
 submit['target'] = target
submit.head()
my_submission = test.copy()
my_submission['ConfirmedCases'] = confirmed
my_submission['Fatalities'] = fatalities
my_submission.head()
my_submission.shape 
<save_to_csv>
submit = submit.drop('target_prob',axis=1)
 submit.to_csv('real-nlp_lstm.csv',index=False )
my_submission[[
'ForecastId', 'ConfirmedCases', 'Fatalities'
]].to_csv('submission.csv', index=False)
 print(DECAY)
 my_submission.head()
my_submission.tail()
my_submission.shape<load_from_csv>
train = pd.read_csv(r'/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv(r'/kaggle/input/nlp-getting-started/test.csv' )
<load_from_csv>
Y = train['target']
train = train.drop('target',axis=1)
 text_data_train = train['text']
text_data_test = test['text']
df = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/train.csv",index_col='Id')
 test = pd.read_csv("/kaggle/input/covid19-global-forecasting-week-1/test.csv")
 e2= set(df['Country/Region'])
 df['ConfirmedCases']=df['ConfirmedCases'].astype(int)
 df['Fatalities']=df['Fatalities'].astype(int)
 df["Date"] = pd.to_datetime(df["Date"])
 df['Weekday']= df.apply(lambda row: row["Date"].weekday() ,axis=1)
 df["Weekday"] =(df["Weekday"] < 5 ).astype(int)
 print(df.shape)
<data_type_conversions>
Y.value_counts()
df['ConfirmedCases']=df['ConfirmedCases'].astype(int)
 df['Fatalities']=df['Fatalities'].astype(int)
 df["Date"] = pd.to_datetime(df["Date"])
 df['Weekday']= df.apply(lambda row: row["Date"].weekday() ,axis=1)
 df["Weekday"] =(df["Weekday"] < 5 ).astype(int )<count_missing_values>
tokenizer = transformers.BertTokenizer.from_pretrained('bert-large-uncased', do_lower_case=True)
 bert_model = transformers.TFBertModel.from_pretrained('bert-large-uncased' )
df.isnull().sum()<groupby>
def bert_encode(data,maximum_length):
input_ids = []
attention_masks = []
for i in range(len(data)) :
encoded = tokenizer.encode_plus(
 data[i],
add_special_tokens=True,
max_length=maximum_length,
pad_to_max_length=True,
return_attention_mask=True, 
)
 input_ids.append(encoded['input_ids'])
 attention_masks.append(encoded['attention_mask'])
 return np.array(input_ids),np.array(attention_masks )
df.groupby(['Date','Country/Region'] ).first()<groupby>
train_input_ids,train_attention_masks = bert_encode(text_data_train,100)
 test_input_ids,test_attention_masks = bert_encode(text_data_test,100 )
df_public = df[df["Date"]<"2020-03-12"]
df_public.groupby(['Date','Lat','Long'], as_index=False ).agg({'ConfirmedCases': 'sum', 'Fatalities': 'sum', 'Weekday': 'first'} )<prepare_x_and_y>
def create_model(bert_model):
input_ids = tf.keras.Input(shape=(100,),dtype='int32')
 attention_masks = tf.keras.Input(shape=(100,),dtype='int32')
 output = bert_model([input_ids,attention_masks])
 output = output[1]
output = tf.keras.layers.Dense(1,activation='sigmoid' )(output)
 model = tf.keras.models.Model(inputs = [input_ids,attention_masks],outputs = output)
 model.compile(Adam(lr=6e-6), loss='binary_crossentropy', metrics=['accuracy'])
 return model
def preprocessing(dataframe):
z=dataframe['Date']-df['Date'].min()
for i in z.index:
z[i]=int(str(z[i] ).split() [0])
 data=dataframe
x =data[['Lat', 'Long', 'Date','Weekday']]
y1 = data[['ConfirmedCases']]
y2 = data[['Fatalities']]
x_test = test[['Lat', 'Long', 'Date']]
x_test["Date"] = pd.to_datetime(x_test["Date"])
 return z,x,y1,y2<feature_engineering>
history = model.fit([train_input_ids,train_attention_masks],Y,
validation_split=0.2,
epochs=3,
batch_size=5 )
z=df_public['Date']-df_public['Date'].min()
for i in z.index:
z[i]=int(str(z[i] ).split() [0] )<prepare_x_and_y>
result = model.predict([test_input_ids,test_attention_masks])
 result = np.round(result ).astype(int)
 submit = pd.DataFrame(test['id'],columns=['id'])
 submit['target'] = result
submit.head()
<data_type_conversions><EOS> 
submit.to_csv('real_nlp_bert.csv',index=False )