Kaggle competition: COVID19 Global Forecasting (Week 4)
embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_3], axis = 0)
 np.shape(embedding_matrix )<predict_on_test>
plt.style.use('fivethirtyeight')
 le = preprocessing.LabelEncoder()
warnings.filterwarnings('ignore' )
outputs = []
pred_val_y, pred_test_y, best_score = train_pred(model_gru_srk_atten(embedding_matrix), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, 'gru atten srk'])
<predict_on_test>
import plotly.io as pio
import plotly.express as px
import plotly.graph_objects as go
pred_val_y, pred_test_y, best_score = train_pred(model_cnn(embedding_matrix_1), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, '2d CNN GloVe'])
<predict_on_test>
train_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 test_df = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 submission = pd.read_csv(".. /input/covid19-global-forecasting-week-4/submission.csv" )
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_atten(embedding_matrix_2), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, best_score, '2 LSTM w/ attention FASTTEXT'] )<predict_on_test>
display(train_df.isnull().sum() /len(train_df)*100)
 display(test_df.isnull().sum() /len(test_df)*100 )
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_du(embedding_matrix), epochs = 2)
 outputs.append([pred_val_y, pred_test_y, best_score, 'LSTM DU'])
<predict_on_test>
train_df.rename(columns={'Province_State':'State','Country_Region':'Country'}, inplace=True)
 test_df.rename(columns={'Province_State':'State','Country_Region':'Country'}, inplace=True )
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_atten(embedding_matrix), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, best_score, '2 LSTM w/ attention'] )<predict_on_test>
def missings(state, country):
return country if pd.isna(state)== True else state
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_atten(embedding_matrix_1), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, best_score, '2 LSTM w/ attention GloVe'])
<predict_on_test>
train_df['State'] = train_df.apply(lambda x: missings(x['State'],x['Country']),axis=1)
 test_df['State'] = test_df.apply(lambda x: missings(x['State'],x['Country']),axis=1 )
pred_val_y, pred_test_y, best_score = train_pred(model_lstm_atten(embedding_matrix_3), epochs = 3)
 outputs.append([pred_val_y, pred_test_y, best_score, '2 LSTM w/ attention Para'] )<sort_values>
print("In our data set are {} countries and {} states.".format(train_df['Country'].nunique() ,train_df['State'].nunique()))
outputs.sort(key=lambda x: x[2])
 weights = [i for i in range(1, len(outputs)+ 1)]
print(weights)
 print()
weights = [float(i)/ sum(weights)for i in weights]
print(weights )<train_model>
df_plot = train_df.loc[: , ['Date', 'Country', 'ConfirmedCases', 'Fatalities']].groupby(['Date', 'Country'] ).max().reset_index()
df_plot.loc[:, 'Size'] = np.power(df_plot["ConfirmedCases"]+1,0.3)-1
fig = px.scatter_geo(df_plot,
locations="Country",
locationmode = "country names",
hover_name="Country",
color="ConfirmedCases",
animation_frame="Date",
size='Size',
projection="natural earth",
title="Global Spread of Covid-19",
width=1500, height=800)
 fig.layout.updatemenus[0].buttons[0].args[1]["frame"]["duration"] = 5
fig.show()
<compute_train_metric>
data_leak = pd.merge(train_df,test_df, how='inner', on='Date')['Date'].unique().tolist()
data_leak.append('2020-04-01')
 data_leak.sort()
print("Both data sets contain the following dates: {}".format(data_leak))
pred_val_y = np.mean([outputs[i][0] for i in range(len(outputs)) ], axis = 0)
 thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh )<save_to_csv>
def create_features(df):
df['Day_num'] = le.fit_transform(df['Date'])
 df['Date'] = pd.to_datetime(df['Date'])
 df['Day'] = df['Date'].dt.day
df['Week'] = df['Date'].dt.week
df['Month'] = df['Date'].dt.month
df['DayOfWeek'] = df['Date'].dt.dayofweek
df['Country'] = le.fit_transform(df['Country'])
 country_dict = dict(zip(le.inverse_transform(df['Country']), df['Country']))
df['State'] = le.fit_transform(df['State'])
 state_dict = dict(zip(le.inverse_transform(df['State']), df['State']))
return df, country_dict, state_dict
pred_test_y = np.mean([outputs[i][1] for i in range(len(outputs)) ], axis = 0)
 pred_test_y =(pred_test_y > best_thresh ).astype(int)
 test_df = pd.read_csv(".. /input/test.csv", usecols=["qid"])
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )<load_from_csv>
df_all, country_dict, state_dict = create_features(df_all )
train = pd.read_csv('.. /input/quora-insincere-questions-classification/train.csv', encoding='utf-8')
 test = pd.read_csv('.. /input/quora-insincere-questions-classification/test.csv', encoding='utf-8')
 df = pd.concat([train,test],axis=0)
 print(f )<load_pretrained>
def train_test_split_extend(df,d,day,filter_col_confirmed,filter_col_fatalities):
df=df.loc[df['Day_num'] >= day]
df_train = df.loc[df['Day_num'] < d]
X_train = df_train
Y_train_1 = df_train['ConfirmedCases']
Y_train_2 = df_train['Fatalities']
X_train_1 = X_train.drop(columns=filter_col_fatalities ).drop(columns='ConfirmedCases')
 X_train_2 = X_train.drop(columns=filter_col_confirmed ).drop(columns='Fatalities')
 df_test = df.loc[df['Day_num'] == d]
x_test = df_test
x_test_1 = x_test.drop(columns=filter_col_fatalities ).drop(columns='ConfirmedCases')
 x_test_2 = x_test.drop(columns=filter_col_confirmed ).drop(columns='Fatalities')
 x_test.drop(['ConfirmedCases', 'Fatalities'], axis=1, inplace=True)
 X_train_1.drop('Id', inplace=True, errors='ignore', axis=1)
 X_train_1.drop('ForecastId', inplace=True, errors='ignore', axis=1)
 X_train_2.drop('Id', inplace=True, errors='ignore', axis=1)
 X_train_2.drop('ForecastId', inplace=True, errors='ignore', axis=1)
 x_test_1.drop('Id', inplace=True, errors='ignore', axis=1)
 x_test_1.drop('ForecastId', inplace=True, errors='ignore', axis=1)
 x_test_2.drop('Id', inplace=True, errors='ignore', axis=1)
 x_test_2.drop('ForecastId', inplace=True, errors='ignore', axis=1)
 return X_train_1, X_train_2, Y_train_1, Y_train_2, x_test_1, x_test_2
def load_embed(file):
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float16')
 if file == '.. /input/quora-insincere-questions-classification/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='utf8')if len(o)>100)
 elif file == '.. /input/quora-insincere-questions-classification/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin':
embeddings_index = KeyedVectors.load_word2vec_format(file, binary=True)
 elif file == '.. /input/quora-insincere-questions-classification/embeddings/paragram_300_sl999/paragram_300_sl999.txt':
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='latin')if len(o)>100)
 else:
embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='latin'))
return embeddings_index
glove = '.. /input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt'
paragram = '.. /input/quora-insincere-questions-classification/embeddings/paragram_300_sl999/paragram_300_sl999.txt' 
<load_pretrained>
def lin_reg(X_train, Y_train, x_test):
regr = linear_model.LinearRegression()
regr.fit(X_train, Y_train)
 pred = regr.predict(x_test)
 return regr, pred
print("Extracting Paragram embedding")
 embed_paragram = load_embed(paragram)
<feature_engineering>
def country_calculation(df_all,country,date,day):
df_country = df_all.copy()
df_country = df_country.loc[df_country['Date'] >= date]
df_country = df_country.loc[df_country['Country'] == country_dict[country]]
features = ['Id', 'State', 'Country','ConfirmedCases', 'Fatalities', 'Day_num']
df_country = df_country[features]
df_country = lag_feature(df_country, 'ConfirmedCases',range(1, 40))
df_country = lag_feature(df_country, 'Fatalities', range(1,20))
filter_col_confirmed = [col for col in df_country if col.startswith('Confirmed')]
filter_col_fatalities= [col for col in df_country if col.startswith('Fataliti')]
filter_col = np.append(filter_col_confirmed, filter_col_fatalities)
 df_country[filter_col] = df_country[filter_col].apply(lambda x: np.log1p(x))
df_country.replace([np.inf, -np.inf], 0, inplace=True)
 df_country.fillna(0, inplace=True)
 start = df_country[df_country['Id']==-999].Day_num.min()
end = df_country[df_country['Id']==-999].Day_num.max()
for d in range(start,end+1):
X_train_1, X_train_2, Y_train_1, Y_train_2, x_test_1, x_test_2 = train_test_split_extend(df_country,d,day,filter_col_confirmed,filter_col_fatalities)
 regr_1, pred_1 = lin_reg(X_train_1, Y_train_1, x_test_1)
 df_country.loc[(df_country['Day_num'] == d)&(df_country['Country'] == country_dict[country]), 'ConfirmedCases'] = pred_1[0]
regr_2, pred_2 = lin_reg(X_train_2, Y_train_2, x_test_2)
 df_country.loc[(df_country['Day_num'] == d)&(df_country['Country'] == country_dict[country]), 'Fatalities'] = pred_2[0]
df_country = lag_feature(df_country, 'ConfirmedCases',range(1, 40))
df_country = lag_feature(df_country, 'Fatalities', range(1,20))
df_country.replace([np.inf, -np.inf], 0, inplace=True)
 df_country.fillna(0, inplace=True)
 print("Calculation done.")
 return df_country
def check_coverage(vocab,embeddings_index):
known_words={}
unknown_words={}
nb_known_words=0
nb_unknown_words=0
for word in vocab.keys() :
if word in embeddings_index.keys() :
known_words[word]=embeddings_index[word]
nb_known_words += vocab[word]
elif word.capitalize() in embeddings_index.keys() :
known_words[word] = embeddings_index[word.capitalize() ]
nb_known_words += vocab[word]
elif word.lower() in embeddings_index.keys() :
known_words[word] = embeddings_index[word.lower() ]
nb_known_words += vocab[word]
elif word.upper() in embeddings_index.keys() :
known_words[word] = embeddings_index[word.upper() ]
nb_known_words += vocab[word]
else:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
print(f'Found embeddings for {round(( len(known_words)/len(vocab)) *100,5)}% of the vocab
Found embeddings for {round(( nb_known_words/(nb_known_words+nb_unknown_words)) *100,5)}% of all text')
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1),reverse=True)
 return unknown_words<define_variables>
def country_state_calculation(df_all,country, state, date,day):
df_country = df_all.copy()
df_country = df_country.loc[df_country['Date'] >= date]
df_country = df_country.loc[df_country['Country'] == country_dict[country] &(df_country['State']==state_dict[state])]
features = ['Id', 'State', 'Country','ConfirmedCases', 'Fatalities', 'Day_num']
df_country = df_country[features]
df_country = lag_feature(df_country, 'ConfirmedCases',range(1, 40))
df_country = lag_feature(df_country, 'Fatalities', range(1,20))
filter_col_confirmed = [col for col in df_country if col.startswith('Confirmed')]
filter_col_fatalities= [col for col in df_country if col.startswith('Fataliti')]
filter_col = np.append(filter_col_confirmed, filter_col_fatalities)
 df_country[filter_col] = df_country[filter_col].apply(lambda x: np.log1p(x))
df_country.replace([np.inf, -np.inf], 0, inplace=True)
 df_country.fillna(0, inplace=True)
 start = df_country[df_country['Id']==-999].Day_num.min()
end = df_country[df_country['Id']==-999].Day_num.max()
for d in range(start,end+1):
X_train_1, X_train_2, Y_train_1, Y_train_2, x_test_1, x_test_2 = train_test_split_extend(df_country,d,day,filter_col_confirmed,filter_col_fatalities)
 regr_1, pred_1 = lin_reg(X_train_1, Y_train_1, x_test_1)
 df_country.loc[(df_country['Day_num'] == d)&(df_country['Country'] == country_dict[country])&(df_country['State'] == state_dict[state]), 'ConfirmedCases'] = pred_1[0]
regr_2, pred_2 = lin_reg(X_train_2, Y_train_2, x_test_2)
 df_country.loc[(df_country['Day_num'] == d)&(df_country['Country'] == country_dict[country]), 'Fatalities'] = pred_2[0]
df_country = lag_feature(df_country, 'ConfirmedCases',range(1, 10))
df_country = lag_feature(df_country, 'Fatalities', range(1,8))
df_country.replace([np.inf, -np.inf], 0, inplace=True)
 df_country.fillna(0, inplace=True)
 print("Calculation done.")
 return df_country
contraction_mapping = {
"Trump's" : 'trump is',"'cause": 'because','â€™': "'",',cause': 'because',';cause': 'because',"ain't": 'am not','ain,t': 'am not',
'ain;t': 'am not','ain´t': 'am not','ain’t': 'am not',"aren't": 'are not','â€“': '-','â€œ':'"',
'aren,t': 'are not','aren;t': 'are not','aren´t': 'are not','aren’t': 'are not',"can't": 'cannot',"can't've": 'cannot have','can,t': 'cannot','can,t,ve': 'cannot have',
'can;t': 'cannot','can;t;ve': 'cannot have',
'can´t': 'cannot','can´t´ve': 'cannot have','can’t': 'cannot','can’t’ve': 'cannot have',
"could've": 'could have','could,ve': 'could have','could;ve': 'could have',"couldn't": 'could not',"couldn't've": 'could not have','couldn,t': 'could not','couldn,t,ve': 'could not have','couldn;t': 'could not',
'couldn;t;ve': 'could not have','couldn´t': 'could not',
'couldn´t´ve': 'could not have','couldn’t': 'could not','couldn’t’ve': 'could not have','could´ve': 'could have',
'could’ve': 'could have',"didn't": 'did not','didn,t': 'did not','didn;t': 'did not','didn´t': 'did not',
'didn’t': 'did not',"doesn't": 'does not','doesn,t': 'does not','doesn;t': 'does not','doesn´t': 'does not',
'doesn’t': 'does not',"don't": 'do not','don,t': 'do not','don;t': 'do not','don´t': 'do not','don’t': 'do not',
"hadn't": 'had not',"hadn't've": 'had not have','hadn,t': 'had not','hadn,t,ve': 'had not have','hadn;t': 'had not',
'hadn;t;ve': 'had not have','hadn´t': 'had not','hadn´t´ve': 'had not have','hadn’t': 'had not','hadn’t’ve': 'had not have',"hasn't": 'has not','hasn,t': 'has not','hasn;t': 'has not','hasn´t': 'has not','hasn’t': 'has not',
"haven't": 'have not','haven,t': 'have not','haven;t': 'have not','haven´t': 'have not','haven’t': 'have not',"he'd": 'he would',
"he'd've": 'he would have',"he'll": 'he will',
"he's": 'he is','he,d': 'he would','he,d,ve': 'he would have','he,ll': 'he will','he,s': 'he is','he;d': 'he would',
'he;d;ve': 'he would have','he;ll': 'he will','he;s': 'he is','he´d': 'he would','he´d´ve': 'he would have','he´ll': 'he will',
'he´s': 'he is','he’d': 'he would','he’d’ve': 'he would have','he’ll': 'he will','he’s': 'he is',"how'd": 'how did',"how'll": 'how will',
"how's": 'how is','how,d': 'how did','how,ll': 'how will','how,s': 'how is','how;d': 'how did','how;ll': 'how will',
'how;s': 'how is','how´d': 'how did','how´ll': 'how will','how´s': 'how is','how’d': 'how did','how’ll': 'how will',
'how’s': 'how is',"i'd": 'i would',"i'll": 'i will',"i'm": 'i am',"i've": 'i have','i,d': 'i would','i,ll': 'i will',
'i,m': 'i am','i,ve': 'i have','i;d': 'i would','i;ll': 'i will','i;m': 'i am','i;ve': 'i have',"isn't": 'is not',
'isn,t': 'is not','isn;t': 'is not','isn´t': 'is not','isn’t': 'is not',"it'd": 'it would',"it'll": 'it will',"It's":'it is',
"it's": 'it is','it,d': 'it would','it,ll': 'it will','it,s': 'it is','it;d': 'it would','it;ll': 'it will','it;s': 'it is','it´d': 'it would','it´ll': 'it will','it´s': 'it is',
'it’d': 'it would','it’ll': 'it will','it’s': 'it is',
'i´d': 'i would','i´ll': 'i will','i´m': 'i am','i´ve': 'i have','i’d': 'i would','i’ll': 'i will','i’m': 'i am',
'i’ve': 'i have',"let's": 'let us','let,s': 'let us','let;s': 'let us','let´s': 'let us',
'let’s': 'let us',"ma'am": 'madam','ma,am': 'madam','ma;am': 'madam',"mayn't": 'may not','mayn,t': 'may not','mayn;t': 'may not',
'mayn´t': 'may not','mayn’t': 'may not','ma´am': 'madam','ma’am': 'madam',"might've": 'might have','might,ve': 'might have','might;ve': 'might have',"mightn't": 'might not','mightn,t': 'might not','mightn;t': 'might not','mightn´t': 'might not',
'mightn’t': 'might not','might´ve': 'might have','might’ve': 'might have',"must've": 'must have','must,ve': 'must have','must;ve': 'must have',
"mustn't": 'must not','mustn,t': 'must not','mustn;t': 'must not','mustn´t': 'must not','mustn’t': 'must not','must´ve': 'must have',
'must’ve': 'must have',"needn't": 'need not','needn,t': 'need not','needn;t': 'need not','needn´t': 'need not','needn’t': 'need not',"oughtn't": 'ought not','oughtn,t': 'ought not','oughtn;t': 'ought not',
'oughtn´t': 'ought not','oughtn’t': 'ought not',"sha'n't": 'shall not','sha,n,t': 'shall not','sha;n;t': 'shall not',"shan't": 'shall not',
'shan,t': 'shall not','shan;t': 'shall not','shan´t': 'shall not','shan’t': 'shall not','sha´n´t': 'shall not','sha’n’t': 'shall not',
"she'd": 'she would',"she'll": 'she will',"she's": 'she is','she,d': 'she would','she,ll': 'she will',
'she,s': 'she is','she;d': 'she would','she;ll': 'she will','she;s': 'she is','she´d': 'she would','she´ll': 'she will',
'she´s': 'she is','she’d': 'she would','she’ll': 'she will','she’s': 'she is',"should've": 'should have','should,ve': 'should have','should;ve': 'should have',
"shouldn't": 'should not','shouldn,t': 'should not','shouldn;t': 'should not','shouldn´t': 'should not','shouldn’t': 'should not','should´ve': 'should have',
'should’ve': 'should have',"that'd": 'that would',"that's": 'that is','that,d': 'that would','that,s': 'that is','that;d': 'that would',
'that;s': 'that is','that´d': 'that would','that´s': 'that is','that’d': 'that would','that’s': 'that is',"there'd": 'there had',
"there's": 'there is','there,d': 'there had','there,s': 'there is','there;d': 'there had','there;s': 'there is',
'there´d': 'there had','there´s': 'there is','there’d': 'there had','there’s': 'there is',
"they'd": 'they would',"they'll": 'they will',"they're": 'they are',"they've": 'they have',
'they,d': 'they would','they,ll': 'they will','they,re': 'they are','they,ve': 'they have','they;d': 'they would','they;ll': 'they will','they;re': 'they are',
'they;ve': 'they have','they´d': 'they would','they´ll': 'they will','they´re': 'they are','they´ve': 'they have','they’d': 'they would','they’ll': 'they will',
'they’re': 'they are','they’ve': 'they have',"wasn't": 'was not','wasn,t': 'was not','wasn;t': 'was not','wasn´t': 'was not',
'wasn’t': 'was not',"we'd": 'we would',"we'll": 'we will',"we're": 'we are',"we've": 'we have','we,d': 'we would','we,ll': 'we will',
'we,re': 'we are','we,ve': 'we have','we;d': 'we would','we;ll': 'we will','we;re': 'we are','we;ve': 'we have',
"weren't": 'were not','weren,t': 'were not','weren;t': 'were not','weren´t': 'were not','weren’t': 'were not','we´d': 'we would','we´ll': 'we will',
'we´re': 'we are','we´ve': 'we have','we’d': 'we would','we’ll': 'we will','we’re': 'we are','we’ve': 'we have',"what'll": 'what will',"what're": 'what are',"what's": 'what is',
"what've": 'what have','what,ll': 'what will','what,re': 'what are','what,s': 'what is','what,ve': 'what have','what;ll': 'what will','what;re': 'what are',
'what;s': 'what is','what;ve': 'what have','what´ll': 'what will',
'what´re': 'what are','what´s': 'what is','what´ve': 'what have','what’ll': 'what will','what’re': 'what are','what’s': 'what is',
'what’ve': 'what have',"where'd": 'where did',"where's": 'where is','where,d': 'where did','where,s': 'where is','where;d': 'where did',
'where;s': 'where is','where´d': 'where did','where´s': 'where is','where’d': 'where did','where’s': 'where is',
"who'll": 'who will',"who's": 'who is','who,ll': 'who will','who,s': 'who is','who;ll': 'who will','who;s': 'who is',
'who´ll': 'who will','who´s': 'who is','who’ll': 'who will','who’s': 'who is',"won't": 'will not','won,t': 'will not','won;t': 'will not',
'won´t': 'will not','won’t': 'will not',"wouldn't": 'would not','wouldn,t': 'would not','wouldn;t': 'would not','wouldn´t': 'would not',
'wouldn’t': 'would not',"you'd": 'you would',"you'll": 'you will',"you're": 'you are','you,d': 'you would','you,ll': 'you will',
'you,re': 'you are','you;d': 'you would','you;ll': 'you will',
'you;re': 'you are','you´d': 'you would','you´ll': 'you will','you´re': 'you are','you’d': 'you would','you’ll': 'you will','you’re': 'you are',
'´cause': 'because','’cause': 'because',"you've": "you have","could'nt": 'could not',
"havn't": 'have not',"here’s": "here is",'i""m': 'i am',"i'am": 'i am',"i'l": "i will","i'v": 'i have',"wan't": 'want',"was'nt": "was not","who'd": "who would",
"who're": "who are","who've": "who have","why'd": "why would","would've": "would have","y'all": "you all","y'know": "you know","you.i": "you i",
"your'e": "you are","arn't": "are not","agains't": "against","c'mon": "common","doens't": "does not",'don""t': "do not","dosen't": "does not",
"dosn't": "does not","shoudn't": "should not","that'll": "that will","there'll": "there will","there're": "there are",
"this'll": "this all","u're": "you are", "ya'll": "you all","you'r": "you are","you’ve": "you have","d'int": "did not","did'nt": "did not","din't": "did not","dont't": "do not","gov't": "government",
"i'ma": "i am","is'nt": "is not","‘I":'I','ᴀɴᴅ':'and','ᴛʜᴇ':'the','ʜᴏᴍᴇ':'home','ᴜᴘ':'up','ʙʏ':'by','ᴀᴛ':'at','…and':'and','civilbeat':'civil beat','TrumpCare':'Trump care','Trumpcare':'Trump care', 'OBAMAcare':'Obama care','ᴄʜᴇᴄᴋ':'check','ғᴏʀ':'for','ᴛʜɪs':'this','ᴄᴏᴍᴘᴜᴛᴇʀ':'computer','ᴍᴏɴᴛʜ':'month','ᴡᴏʀᴋɪɴɢ':'working','ᴊᴏʙ':'job','ғʀᴏᴍ':'from','Sᴛᴀʀᴛ':'start','gubmit':'submit','CO₂':'carbon dioxide','ғɪʀsᴛ':'first','ᴇɴᴅ':'end','ᴄᴀɴ':'can','ʜᴀᴠᴇ':'have','ᴛᴏ':'to','ʟɪɴᴋ':'link','ᴏғ':'of','ʜᴏᴜʀʟʏ':'hourly','ᴡᴇᴇᴋ':'week','ᴇɴᴅ':'end','ᴇxᴛʀᴀ':'extra','Gʀᴇᴀᴛ':'great','sᴛᴜᴅᴇɴᴛs':'student','sᴛᴀʏ':'stay','ᴍᴏᴍs':'mother','ᴏʀ':'or','ᴀɴʏᴏɴᴇ':'anyone','ɴᴇᴇᴅɪɴɢ':'needing','ᴀɴ':'an','ɪɴᴄᴏᴍᴇ':'income',
'ʀᴇʟɪᴀʙʟᴇ':'reliable','ғɪʀsᴛ':'first','ʏᴏᴜʀ':'your','sɪɢɴɪɴɢ':'signing','ʙᴏᴛᴛᴏᴍ':'bottom','ғᴏʟʟᴏᴡɪɴɢ':'following','Mᴀᴋᴇ':'make',
'ᴄᴏɴɴᴇᴄᴛɪᴏɴ':'connection','ɪɴᴛᴇʀɴᴇᴛ':'internet','financialpost':'financial post', 'ʜaᴠᴇ':' have ', 'ᴄaɴ':' can ', 'Maᴋᴇ':' make ', 'ʀᴇʟɪaʙʟᴇ':' reliable ', 'ɴᴇᴇᴅ':' need ',
'ᴏɴʟʏ':' only ', 'ᴇxᴛʀa':' extra ', 'aɴ':' an ', 'aɴʏᴏɴᴇ':' anyone ', 'sᴛaʏ':' stay ', 'Sᴛaʀᴛ':' start', 'SHOPO':'shop',
}<define_variables>
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }<define_variables>
!pip install pyramid.arima
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization', 'lnmiit':'limit', 'quorans': 'users of quora', 'mhcet':'competitive examination', 'swachh':'clean', 'chapterwise':'chapter wise', 'bitconnect':'cryptocurrency', 'bittrex':'cryptocurrency exchange','koinex':'cryptocurrency exchange', 'zebpay':'cryptocurrency exchange', 'binance':'cryptocurrency exchange', 'coinbase':'cryptocurrency exchange', 'brexit':'exit of UK from European Union', 'cryptocurrencies':'multiple cryptocurrency','redmi':'mobile phone', 'oneplus':'mobile phone company', 'pokémon':'anime', 'boruto':'anime', 'bhakts':'loyal followers', 'litecoin':'cryptocurreny','qoura':'quora','altcoin':'cryptocurreny','blockchains':'blockchain', 'airpods':'wireless earphones','zenfone':'mobile phone','altcoins':'multiple cryptocurrency','electroneum':'cryptocurreny','cryptocurreny':'blockchain based currency','iitians':'students','iitian':'student','gdpr':'data protection law'}<feature_engineering>
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)))
 pd.set_option('mode.chained_assignment', None)
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 train['Province_State'].fillna('', inplace=True)
 test['Province_State'].fillna('', inplace=True)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date'] = pd.to_datetime(test['Date'])
 train = train.sort_values(['Country_Region','Province_State','Date'])
 test = test.sort_values(['Country_Region','Province_State','Date'] )
train['treated_question'] = train['question_text'].apply(lambda x: x.lower())
 test['treated_question'] = test['question_text'].apply(lambda x: x.lower())
 train['treated_question'] = train['treated_question'].apply(lambda x: clean_contractions(x, contraction_mapping))
test['treated_question'] = test['treated_question'].apply(lambda x: clean_contractions(x, contraction_mapping))
train['treated_question'] = train['treated_question'].apply(lambda x: clean_puncts(x, punct_mapping, punct))
test['treated_question'] = test['treated_question'].apply(lambda x: clean_puncts(x, punct_mapping, punct))
train['treated_question'] = train['treated_question'].apply(lambda x: correct_spell(x, mispell_dict))
test['treated_question'] = test['treated_question'].apply(lambda x: correct_spell(x, mispell_dict))<import_modules>
feature_day = [1,20,50,100,200,500,1000]
def CreateInput(data):
feature = []
for day in feature_day:
data.loc[:,'Number day from ' + str(day)+ ' case'] = 0
if(train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].count() > 0):
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['ConfirmedCases'] < day)]['Date'].max()
else:
fromday = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].min()
for i in range(0, len(data)) :
if(data['Date'].iloc[i] > fromday):
day_denta = data['Date'].iloc[i] - fromday
data['Number day from ' + str(day)+ ' case'].iloc[i] = day_denta.days
feature = feature + ['Number day from ' + str(day)+ ' case']
return data[feature]
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.models import Model, Sequential
from keras.layers import Flatten,Dense, Embedding, Conv1D, MaxPooling1D, GlobalAveragePooling1D, GlobalMaxPooling1D, Concatenate, Input, Dropout,Bidirectional, CuDNNGRU, GlobalAveragePooling1D, GlobalMaxPooling1D, LeakyReLU, Activation, CuDNNLSTM, SpatialDropout1D
from keras.optimizers import Adam
from keras import backend as K
from sklearn.model_selection import train_test_split
from sklearn.metrics import precision_recall_fscore_support<define_variables>
pred_data_all = pd.DataFrame()
for country in train['Country_Region'].unique() :
for province in train[(train['Country_Region'] == country)]['Province_State'].unique() :
print(country + ' and ' + province)
 df_train = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]
df_test = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
X_train = CreateInput(df_train)
 y_train_confirmed = df_train['ConfirmedCases'].ravel()
y_train_fatalities = df_train['Fatalities'].ravel()
X_pred = CreateInput(df_test)
 for day in sorted(feature_day,reverse = True):
feature_use = 'Number day from ' + str(day)+ ' case'
idx = X_train[X_train[feature_use] == 0].shape[0]
if(X_train[X_train[feature_use] > 0].shape[0] >= 20):
break
adjusted_X_train = X_train[idx:][feature_use].values.reshape(-1, 1)
 adjusted_y_train_confirmed = y_train_confirmed[idx:]
adjusted_y_train_fatalities = y_train_fatalities[idx:]
idx = X_pred[X_pred[feature_use] == 0].shape[0]
adjusted_X_pred = X_pred[idx:][feature_use].values.reshape(-1, 1)
 pred_data = test[(test['Country_Region'] == country)&(test['Province_State'] == province)]
max_train_date = train[(train['Country_Region'] == country)&(train['Province_State'] == province)]['Date'].max()
min_test_date = pred_data['Date'].min()
if len(adjusted_y_train_confirmed)< 1:
adjusted_y_train_confirmed = np.zeros(3)
 else:
if len(adjusted_y_train_confirmed)< 2:
adjusted_y_train_confirmed = np.append(adjusted_y_train_confirmed,adjusted_y_train_confirmed[len(adjusted_y_train_confirmed)-1],adjusted_y_train_confirmed[len(adjusted_y_train_confirmed)-1])
 else:
if len(adjusted_y_train_confirmed)< 3:
adjusted_y_train_confirmed = np.append(adjusted_y_train_confirmed,adjusted_y_train_confirmed[len(adjusted_y_train_confirmed)-1])
 else:
pass
model = SARIMAX(adjusted_y_train_confirmed, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_confirmed = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_confirmed = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['ConfirmedCases'].values
y_hat_confirmed = np.concatenate(( y_train_confirmed,y_hat_confirmed), axis = 0)
 if len(adjusted_y_train_fatalities)< 1:
adjusted_y_train_fatalities = np.zeros(3)
 else:
if len(adjusted_y_train_fatalities)< 2:
adjusted_y_train_fatalities = np.append(adjusted_y_train_fatalities,adjusted_y_train_fatalities[len(adjusted_y_train_fatalities)-1],adjusted_y_train_fatalities[len(adjusted_y_train_fatalities)-1])
 else:
if len(adjusted_y_train_fatalities)< 3:
adjusted_y_train_fatalities = np.append(adjusted_y_train_fatalities,adjusted_y_train_fatalities[len(adjusted_y_train_fatalities)-1])
 else:
pass
model = SARIMAX(adjusted_y_train_fatalities, order=(1,1,0),
measurement_error=True ).fit(disp=False)
 y_hat_fatalities = model.forecast(pred_data[pred_data['Date'] > max_train_date].shape[0])
 y_train_fatalities = train[(train['Country_Region'] == country)&(train['Province_State'] == province)&(train['Date'] >= min_test_date)]['Fatalities'].values
y_hat_fatalities = np.concatenate(( y_train_fatalities,y_hat_fatalities), axis = 0)
 pred_data['ConfirmedCases_hat'] = y_hat_confirmed
pred_data['Fatalities_hat'] = y_hat_fatalities
pred_data_all = pred_data_all.append(pred_data )
length_to_keep = 100000
max_len = 75<define_variables>
df_val = pd.merge(pred_data_all,train[['Date','Country_Region','Province_State','ConfirmedCases','Fatalities']],on=['Date','Country_Region','Province_State'], how='left')
 df_val.loc[df_val['Fatalities_hat'] < 0,'Fatalities_hat'] = 0
df_val.loc[df_val['ConfirmedCases_hat'] < 0,'ConfirmedCases_hat'] = 0
df_val_3 = df_val.copy()
submission = df_val[['ForecastId','ConfirmedCases_hat','Fatalities_hat']]
submission.columns = ['ForecastId','ConfirmedCases','Fatalities']
submission.to_csv('submission.csv', index=False )
<string_transform>
def tokenize_treated_data(text, text_test):
t = Tokenizer(num_words=length_to_keep, filters='')
 t.fit_on_texts(text)
 text = t.texts_to_sequences(text)
 text_test = t.texts_to_sequences(text_test)
 text = pad_sequences(text, maxlen=max_len)
 text_test = pad_sequences(text_test, maxlen=max_len)
 return text, t.word_index, text_test
X_treated, word_index_treated, X_test_treated = tokenize_treated_data(train['treated_question'], test['treated_question'] )<categorify>
def crosscheck_sarima(country):
crosscheck = train_df[(train_df['Country'] == country)&(train_df['Date'] >= '2020-04-02')].reset_index()
arima = pred_data_all[(pred_data_all['Country_Region'] == country)].reset_index()
arima['ConfirmedCases_In'] = arima['ConfirmedCases_hat']
arima['Fatalities_In'] = arima['Fatalities_hat']
arima['CC_Crosscheck'] = crosscheck['ConfirmedCases']
arima['Fat_Crosscheck'] = crosscheck['Fatalities']
arima['Day_num'] = arima['Date']
return arima
vocab_len_treated = len(word_index_treated)+1
def embed_matrix(embed_paragram, word_index, length_to_keep):
embeddings = np.stack(embed_paragram.values())
 embeddings_shape = embeddings.shape[1]
embedding_matrix = np.zeros(( length_to_keep, 300))
for word, i in word_index.items() :
if i >= length_to_keep:
continue
embeddings_vector = embed_paragram.get(word)
 if embeddings_vector is not None:
embedding_matrix[i] = embeddings_vector
return embedding_matrix
embedding_treated = embed_matrix(embed_paragram, word_index_treated, length_to_keep)
<drop_column>
def crosscheck_sarima_cs(country,state):
crosscheck = train_df[(train_df['Country'] == country)&(train_df['State'] == state)&(train_df['Date'] >= '2020-04-02')].reset_index()
arima = pred_data_all[(pred_data_all['Country_Region'] == country)&(pred_data_all['Province_State'] == state)].reset_index()
arima['ConfirmedCases_In'] = arima['ConfirmedCases_hat']
arima['Fatalities_In'] = arima['Fatalities_hat']
arima['CC_Crosscheck'] = crosscheck['ConfirmedCases']
arima['Fat_Crosscheck'] = crosscheck['Fatalities']
arima['Day_num'] = arima['Date']
return arima