Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<feature_engineering>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib.pyplot import xticks
from nltk.corpus import stopwords
import nltk
import re
from nltk.stem import WordNetLemmatizer
import string
from nltk.tokenize import word_tokenize
from nltk.util import ngrams
from collections import defaultdict
from tqdm import tqdm
from sklearn.model_selection import train_test_split
from keras.utils.vis_utils import plot_model
from keras.models import Sequential
from keras.layers import Dense
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, Callback
import tensorflow as tf
from sklearn.metrics import f1_score
from wordcloud import WordCloud,STOPWORDS
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.pipeline import Pipeline
from keras.preprocessing.sequence import pad_sequences
from numpy import array
from numpy import asarray
from numpy import zeros
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential
from keras.layers import Dense,Flatten,Embedding,Activation,Dropout
from keras.layers import Conv1D,MaxPooling1D,GlobalMaxPooling1D,LSTM
c=z.max() +1
y=x_test['Date']-x_test['Date'].min()
for i in y.index:
y[i]=int(str(y[i] ).split() [0])+c<feature_engineering>
train= pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test=pd.read_csv('.. /input/nlp-getting-started/test.csv' )
x['Date']=z
x_test['Date']=y<drop_column>
train.isnull().sum().sort_values(ascending = False )
x_test2=x_test.drop(['Weekday'],axis=1)
 x2=x.drop(['Weekday'],axis=1 )<import_modules>
print("No.of Real Disaster Tweets(Target = 1):",len(train[train["target"]==1]))
print("No.of Fake Disaster Tweets(Target = 0):",len(train[train["target"]==0]))
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import KFold, cross_val_score, train_test_split<train_model>
def length(text):
return len(text)
 train["length"]= train.text.apply(length )
scaler = StandardScaler()
X_scaled = scaler.fit_transform(x)
 x_test_scaled = scaler.transform(x_test)
 X_train, X_test, y_train, y_test = train_test_split(X_scaled, y1, test_size=0.2, random_state=44 )<choose_model_class>
train.drop("length",1,inplace=True )
models = []
models.append(( "RF",RandomForestClassifier()))
models.append(( "Dtree",DecisionTreeClassifier()))
models.append(( "KNN",KNeighborsClassifier()))<compute_train_metric>
stop = list(stopwords.words("english"))
for name,model in models:
kfold = KFold(n_splits=2, random_state=22)
 cv_result = cross_val_score(model,X_train,y_train, cv = kfold,scoring = "accuracy")
 print(name, cv_result )<split>
sw = []
for message in train.text:
for word in message.split() :
if word in stop:
sw.append(word)
 wordlist = nltk.FreqDist(sw)
 top10 = wordlist.most_common(10 )
X_train, X_test, y_train, y_test = train_test_split(X_scaled, y2, test_size=0.2, random_state=44 )<compute_train_metric>
punctuation = list(string.punctuation )
for name,model in models:
kfold = KFold(n_splits=2, random_state=22)
 cv_result = cross_val_score(model,X_train,y_train, cv = kfold,scoring = "accuracy")
 print(name, cv_result )<prepare_x_and_y>
pun = []
for message in train.text:
for word in message.split() :
if word in punctuation:
pun.append(word)
 wordlist = nltk.FreqDist(pun)
 top10 = wordlist.most_common(10 )
z,x,y1,y2 = preprocessing(df )<feature_engineering>
stop_real = []
pun_real = []
for message in train[train.target==1]["text"]:
for word in message.split() :
if word in stop:
stop_real.append(word)
 if word in punctuation:
pun_real.append(word)
 stop_real_wordlist = nltk.FreqDist(stop_real)
 pun_real_wordlist = nltk.FreqDist(pun_real)
 stop_real_top10 = stop_real_wordlist.most_common(10)
 pun_real_top10 = pun_real_wordlist.most_common(10 )
x['Date']=z<categorify>
stop_fake = []
pun_fake = []
for message in train[train.target==0]["text"]:
for word in message.split() :
if word in stop:
stop_fake.append(word)
 if word in punctuation:
pun_fake.append(word)
 stop_fake_wordlist = nltk.FreqDist(stop_fake)
 pun_fake_wordlist = nltk.FreqDist(pun_fake)
 stop_fake_top10 = stop_fake_wordlist.most_common(10)
 pun_fake_top10 = pun_fake_wordlist.most_common(10 )
print("-----------Confirmed------
")
 do_your_thing(x,y1)
 print("-----------Fatality------
")
 do_your_thing(x,y2 )<split>
lemma = WordNetLemmatizer()
kfold = KFold(n_splits=2, shuffle = True, random_state = 0)
 scaler = StandardScaler()
X_scaled = scaler.fit_transform(x)
 X_train1, X_test1, y_train1, y_test1 = train_test_split(X_scaled, y1, test_size=0.2, random_state=44)
 best_score = 0
for n_estimators in [90, 100, 110, 120, 200]:
for max_features in [0.6, 0.7, 0.8, 0.9, 1]:
print(n_estimators,"--",max_features,"--",'
')
 rf = RandomForestClassifier(n_estimators = n_estimators, criterion = 'entropy', max_features = max_features)
 scores = cross_val_score(rf, X_train1, y_train1, cv=kfold)
 score = np.mean(scores)
 print("Score is ",score,"
")
 if score > best_score:
best_score = score
best_parameters = {'n_estimators': n_estimators, 'max_features': max_features,'criterion':'entropy'}
print("BREACHED!!-->",best_parameters,"
" )<train_model>
sw_pun = stop + punctuation
print("Params: ", best_parameters )<normalization>
def preprocess(tweet):
tweet = re.sub(r"https?:\/\/t.co\/[A-Za-z0-9]+", "", tweet)
 tweet = re.sub('[^\w]',' ',tweet)
 tweet = re.sub('[\d]','',tweet)
 tweet = tweet.lower()
words = tweet.split()
sentence = ""
for word in words:
if word not in(sw_pun):
word = lemma.lemmatize(word,pos = 'v')
 if len(word)> 3:
sentence = sentence + word + ' '
return(sentence )
scaler = StandardScaler()
X_scaled = scaler.fit_transform(x)
<train_model>
train['text'] = train['text'].apply(lambda s : preprocess(s))
test ['text'] = test ['text'].apply(lambda s : preprocess(s))
rf = RandomForestClassifier(**best_parameters)
 rf.fit(X_scaled,y1 )<normalization>
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text )
test_scaled = scaler.transform(x_test )<predict_on_test>
train['text'] = train['text'].apply(lambda s : remove_emoji(s))
test ['text'] = test ['text'].apply(lambda s : remove_emoji(s))
y_pred_confirmed = rf.predict(test_scaled )<prepare_output>
def create_vocab(df):
vocab = Counter()
for i in range(df.shape[0]):
vocab.update(df.text[i].split())
 return(vocab)
predictions = pd.DataFrame({'ForecastId':test['ForecastId'],'ConfirmedCases':y_pred_confirmed})
 predictions.head()<train_model>
master=pd.concat(( train,test)).reset_index(drop=True)
 vocab = create_vocab(master)
 len(vocab )
print("Params: ", best_parameters )<train_model>
vocab.most_common(50)
scaler = StandardScaler()
X_scaled = scaler.fit_transform(x)
 rf = RandomForestClassifier(**best_parameters)
 rf.fit(X_scaled,y2 )<normalization>
final_vocab = []
min_occur = 2
for k,v in vocab.items() :
if v >= min_occur:
final_vocab.append(k )
test_scaled = scaler.transform(x_test )<predict_on_test>
def filter(tweet):
sentence = ""
for word in tweet.split() :
if word in final_vocab:
sentence = sentence + word + ' '
return(sentence )
y_pred_fatal = rf.predict(test_scaled )<prepare_output>
train['text'] = train['text'].apply(lambda s : filter(s))
test ['text'] = test ['text'].apply(lambda s : filter(s))
predictions = pd.DataFrame({'ForecastId':test['ForecastId'],'ConfirmedCases':y_pred_confirmed,'Fatalities':y_pred_fatal})
 predictions.head()<train_model>
real = train[train.target==1].reset_index()
fake = train[train.target==0].reset_index()
')
 ')
<predict_on_test>
def get_ngrams(data,n):
all_words = []
for i in range(len(data)) :
temp = data["text"][i].split()
for word in temp:
all_words.append(word)
 tokenized = all_words
esBigrams = ngrams(tokenized, n)
 esBigram_wordlist = nltk.FreqDist(esBigrams)
 top100 = esBigram_wordlist.most_common(100)
 top100 = dict(top100)
 df_ngrams = pd.DataFrame(sorted(top100.items() , key=lambda x: x[1])[::-1])
 return df_ngrams
<save_to_csv>
real_unigrams = get_ngrams(real,1)
 fake_unigrams = get_ngrams(fake,1 )
predictions.to_csv('submission.csv', header=True, index=False )<import_modules>
real_bigrams = get_ngrams(real,2)
 fake_bigrams = get_ngrams(fake,2 )
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
data_dir = Path('.. /input/covid19-global-forecasting-week-1')
 pio.templates.default = 'ggplot2'<categorify>
real_trigrams = get_ngrams(real,3)
 fake_trigrams = get_ngrams(fake,3 )
<load_from_csv>
def word_cloud(df):
comment_words = ''
stopwords = set(STOPWORDS)
 for val in df.text:
val = str(val)
 tokens = val.split()
for i in range(len(tokens)) :
tokens[i] = tokens[i].lower()
comment_words += " ".join(tokens)+" "
wordcloud = WordCloud(width = 800, height = 800,
background_color ='white',
stopwords = stopwords,
min_font_size = 10 ).generate(comment_words)
 plt.figure(figsize =(8, 8), facecolor = None)
 plt.imshow(wordcloud)
 plt.axis("off")
 plt.tight_layout(pad = 0)
 plt.show()
data = pd.read_csv(data_dir/'train.csv')
 data_test = pd.read_csv(data_dir/'test.csv')
<feature_engineering>
def get_f1(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 recall = true_positives /(possible_positives + K.epsilon())
 f1_val = 2*(precision*recall)/(precision+recall+K.epsilon())
 return f1_val
day_min = pd.to_datetime(data['Date'].min() ,format='%Y-%m-%d')
 t =(pd.DatetimeIndex(data['Date'])- day_min ).days
pd.DatetimeIndex(data['Date'] ).dayofweek<data_type_conversions>
def create_tokenizer(lines):
tokenizer = Tokenizer()
tokenizer.fit_on_texts(lines)
 return tokenizer
l_use_date_int = True
l_use_days_num = True
methods = ['use_date_int','use_days_num','use_converter']
c_method = methods[1]
if c_method == methods[0]:
data["Date"] = data["Date"].apply(lambda x: x.replace("-",""))
data["Date"] = data["Date"].astype(int)
 data_test["Date"] = data_test["Date"].apply(lambda x: x.replace("-",""))
data_test["Date"] = data_test["Date"].astype(int)
 features = ['Lat','Long','Date']
elif c_method == methods[1]:
day_min = pd.to_datetime(data['Date'].min() ,format='%Y-%m-%d')
 t =(pd.DatetimeIndex(data['Date'])- day_min ).days
data.loc[:,'days'] = pd.Series(t)
 t =(pd.DatetimeIndex(data_test['Date'])- day_min ).days
data_test.loc[:,'days'] = pd.Series(t)
 features = ['Lat','Long','days']
elif c_method == methods[2]:
t =(pd.DatetimeIndex(data['Date'])- pd.DatetimeIndex([data_test['Date'].min() ])[0] ).days
data['year'] = pd.DatetimeIndex(data['Date'] ).year
data['month'] = pd.DatetimeIndex(data['Date'] ).month
data['day'] = pd.DatetimeIndex(data['Date'] ).day
data_test['year'] = pd.DatetimeIndex(data_test['Date'] ).year
data_test['month'] = pd.DatetimeIndex(data_test['Date'] ).month
data_test['day'] = pd.DatetimeIndex(data_test['Date'] ).day
features = ['Lat','Long','year','month','day']
features += ['dayofweek']
data['dayofweek'] = pd.DatetimeIndex(data['Date'] ).dayofweek
data_test['dayofweek'] = pd.DatetimeIndex(data_test['Date'] ).dayofweek
X = data[features]
X1 = data[features]
y = data.ConfirmedCases
y1 = data.Fatalities
X_test = data_test[features]
X_train, X_valid, y_train, y_valid = train_test_split(X, y,
train_size=0.8, test_size=0.2,
random_state=0)
 X1_train, X1_valid, y1_train, y1_valid = train_test_split(X1, y1,
train_size=0.8, test_size=0.2,
random_state=0)
<count_unique_values>
X = train.text
y = train.target
test_id = test.id
test.drop(["id","location","keyword"],1,inplace = True )
categorical_cols = [cname for cname in data.columns if
data[cname].nunique() < 10 and
data[cname].dtype == "object"]
numerical_cols = [cname for cname in X.columns if
X[cname].dtype in ['int64', 'float64']]
print(categorical_cols, numerical_cols )<count_unique_values>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 42 )
[cname for cname in data.columns]
for cname in data.columns:
print(cname, data[cname].nunique() < 10, data[cname].dtype == "object" )<choose_model_class>
tokenizer = create_tokenizer(X_train)
 X_train_set = tokenizer.texts_to_matrix(X_train, mode = 'freq')
model0 = RandomForestRegressor(n_estimators=110, random_state=0)
 model1 = RandomForestRegressor(n_estimators=50, random_state=0)
 model0 = DecisionTreeClassifier(criterion='entropy')
 model1 = DecisionTreeClassifier(criterion='entropy')
 model0 = DecisionTreeRegressor(random_state = 0)
 model1 = DecisionTreeRegressor(random_state = 0)
 my_pipeline0 = Pipeline(steps=[ 
('preprocessor', SimpleImputer()), 
('model', model0)
 ])
 my_pipeline1 = Pipeline(steps=[ 
('preprocessor', SimpleImputer()), 
('model', model1)
 ] )<compute_test_metric>
def define_model(n_words):
model = Sequential()
model.add(Dense(128, input_shape=(n_words,), activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics = [get_f1])
 model.summary()
plot_model(model, to_file='model.png', show_shapes=True)
 return model
scores = -1 * cross_val_score(my_pipeline0, X, y,
cv=5,
scoring='neg_mean_absolute_error')
 print("Average MAE score:", scores.mean() )<compute_train_metric>
model.fit(X_train_set,y_train,epochs=10,verbose=2 )
scores = -1 * cross_val_score(my_pipeline1, X, y1,
cv=5,
scoring='neg_mean_absolute_error')
 print("Average MAE score:", scores.mean() )<compute_train_metric>
X_test_set = tokenizer.texts_to_matrix(X_test, mode = 'freq')
 y_pred = model.predict_classes(X_test_set )
def get_score(yy, n_estimators):

 my_pipeline = Pipeline(steps=[ 
('preprocessor', SimpleImputer()), 
('model',RandomForestRegressor(n_estimators=n_estimators,random_state=0))
])
 score_cross_valids = -1 * cross_val_score(my_pipeline, X, yy,
cv=3,
scoring='neg_mean_absolute_error')
 return score_cross_valids.mean() 
<train_model>
print(classification_report(y_test, y_pred))
my_pipeline0.fit(X, y)
 my_pipeline1.fit(X, y1)
<predict_on_test>
test_set = tokenizer.texts_to_matrix(test.text, mode = 'freq' )
rf_predictions = my_pipeline0.predict(X_valid)
 rf_val_mae = mean_absolute_error(rf_predictions, y_valid)
 print("Validation MAE for Random Forest Model: {}".format(rf_val_mae)) 
<predict_on_test>
y_test_pred = model.predict_classes(test_set )
test_preds0 = my_pipeline0.predict(X_test)
 test_preds1 = my_pipeline1.predict(X_test)
 t0 = np.round(test_preds0 ).astype(int)
 t1 = np.round(test_preds1 ).astype(int)
<save_to_csv>
sub = pd.DataFrame()
sub['Id'] = test_id
sub['target'] = y_test_pred
sub.to_csv('submission_1.csv',index=False )
output = pd.DataFrame({'ForecastId': data_test.ForecastId,
'ConfirmedCases':t0,
'Fatalities': t1})
 output.to_csv('submission.csv', index=False )<feature_engineering>
t = Tokenizer()
t.fit_on_texts(X_train.tolist() )
data['log_ConfirmedCases'] = np.log(data['ConfirmedCases']+1)
 data['log_Fatalities'] = np.log(data['Fatalities']+1)
 y_pred2 = data['log_ConfirmedCases']
y1_pred2 = data['log_Fatalities']
day_min = pd.to_datetime(data['Date'].min() ,format='%Y-%m-%d')
 t = data['days'] < data_test['days'].min()
X_train = X[t]
X_valid = X[~t]
y_train = y_pred2[t]
y_valid = y_pred2[~t] 
<choose_model_class>
vocab_size = len(t.word_index)+ 1
model0 = RandomForestRegressor(n_estimators=300, random_state=0,verbose=True)
 model1 = RandomForestRegressor(n_estimators=100, random_state=0)
 my_pipeline0 = Pipeline(steps=[ 
('preprocessor', SimpleImputer()), 
('model', model0)
 ])
 my_pipeline1 = Pipeline(steps=[ 
('preprocessor', SimpleImputer()), 
('model', model1)
 ])
<train_model>
embeddings_index = dict()
f = open('.. /input/glove6b100dtxt/glove.6B.100d.txt', mode='rt', encoding='utf-8')
 for line in f:
values = line.split()
word = values[0]
coefs = asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f.close()
print('Loaded %s word vectors.' % len(embeddings_index))
my_pipeline0.fit(X_train, y_train )<predict_on_test>
encoded_docs = t.texts_to_sequences(X_train.tolist())
 max_length = 100
padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post')
 print(padded_docs )
rf_predictions = my_pipeline0.predict(X_valid)
 rf_val_mae = mean_absolute_error(rf_predictions, y_valid)
 print("Validation MAE for Random Forest Model: {}".format(rf_val_mae))<compute_train_metric>
mis_spelled = []
embedding_matrix = zeros(( vocab_size, 100))
for word, i in t.word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
else:
mis_spelled.append(word )
def get_score1(n_est):
model = RandomForestRegressor(n_estimators=n_est, random_state=0)
 my_pipeline = Pipeline(steps=[ 
('preprocessor', SimpleImputer()), 
('model', model)
 ])
 score_cross_valids = -1 * cross_val_score(my_pipeline0, X_train, y_train,
cv=5,
scoring='neg_mean_absolute_error')
 return score_cross_valids.mean()<train_model>
model = Sequential()
e = Embedding(vocab_size, 100, weights=[embedding_matrix], input_length=100, trainable=False)
 model.add(e)
 model.add(Flatten())
 model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[get_f1])
 model.summary()
model.fit(padded_docs, y_train, epochs=50, verbose=0 )
my_pipeline0.fit(X, y_pred2)
 my_pipeline1.fit(X, y1_pred2)
<predict_on_test>
loss, accuracy = model.evaluate(padded_docs, y_train, verbose=0 )
test_preds2 = my_pipeline0.predict(X_test)
 test_preds3 = my_pipeline1.predict(X_test)
 t0 = np.round(np.exp(test_preds2)-1 ).astype(int)
 t1 = np.round(np.exp(test_preds3)-1 ).astype(int)
<save_to_csv>
encoded_docs = t.texts_to_sequences(X_test.tolist())
 padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post' )
output = pd.DataFrame({'ForecastId': data_test.ForecastId,
'ConfirmedCases':t0,
'Fatalities': t1})
 output.to_csv('submission.csv', index=False )<compute_test_metric>
y_pred = model.predict_classes(padded_docs )
%matplotlib inline
def sigmoid_sqrt_func(x, a, b, c, d, e):
return c + d /(1.0 + np.exp(-a*x+b)) + e*x**0.5
def sigmoid_linear_func(x, a, b, c, d, e):
return c + d /(1.0 + np.exp(-a*x+b)) + e*0.1*x
def sigmoid_quad_func(x, a, b, c, d, e, f):
return c + d /(1.0 + np.exp(-a*x+b)) + e*0.1*x + f*0.001*x*x
def sigmoid_func(x, a, b, c, d):
return c + d /(1.0 + np.exp(-a*x+b))
def exp_func(x, a, b, c, d):
return c + d * np.exp(a*x+b)
 def func_fitting(y, func=sigmoid_func, x_scale=50.0, y_scale=10000.0, start_pred=8, AN=0, MAXN=60, PN=15, b=5):
x = range(len(y))
x_real = np.array(x)/x_scale
y_real = np.array(y)/y_scale
x_train = x_real
y_train = y_real
def next_day_pred(AN, BN):
x_train = x_real[AN:BN]
y_train = y_real[AN:BN]
popt, pcov = curve_fit(func, x_train, y_train,
method='trf',
maxfev=20000,
p0=(1, 0, 0, 1),
bounds=[(-b, -np.inf, -np.inf, -b),(b, np.inf, np.inf, b)], 
)
 x_pred = np.array(range(MAXN)) /x_scale
y_pred = func(x_pred, *popt)
 return x_pred, y_pred
NP = start_pred
y_pred = [np.nan]*NP
y_pred_list = []
for BN in range(NP, len(y_real)) :
x_pred, y_pred_ = next_day_pred(BN-PN, BN)
 y_pred.append(y_pred_[BN])
 y_pred_list.append(y_pred_)
 for BN in range(len(y_real), len(y_pred_)) :
y_pred.append(y_pred_[BN])
 y_pred = np.array(y_pred)
 y_pred_list = np.array(y_pred_list)
 y_pred_std = np.std(y_pred_list[-2:], axis=0)
 return x_real*x_scale, y_real*y_scale, x_train*x_scale, y_train*y_scale, \
x_pred*x_scale, y_pred*y_scale, y_pred_std*y_scale
def draw_figure(start_date, title, x_real, y_real, x_train, y_train, x_pred, y_pred, y_pred_std):
def to_date(idx):
idx = np.round(idx)
 return datetime.datetime.strptime(start_date, '%m/%d/%Y' ).date() + datetime.timedelta(days=idx)
 fig, ax1 = plt.subplots(figsize=[14, 7])
 plot1 = ax1.plot(list(map(to_date, x_real)) , y_real, 'gs',label='original')
 plot2 = ax1.plot(list(map(to_date, x_pred)) , y_pred, 'r',label='predict')
 plot3 = ax1.fill_between(list(map(to_date, x_pred)) ,
np.maximum(0,(y_pred-y_pred_std)) , 
(y_pred+y_pred_std),
alpha=0.2, edgecolor='
plot0 = ax1.plot(list(map(to_date, x_train)) , y_train, 'y.',label='history')
 ax2=ax1.twinx()
ax2.plot(list(map(to_date, x_real)) [1:],(y_real[1:]-y_real[:-1]), '-s',label='original add')
 ax2.plot(list(map(to_date, x_pred)) [1:],(y_pred[1:]-y_pred[:-1]), '-',label='pred add')
 plt.gca().xaxis.set_major_formatter(mdates.DateFormatter('%m-%d'))
plt.gca().xaxis.set_major_locator(mdates.DayLocator())
 plt.gcf().autofmt_xdate()
plt.xlabel('x')
 plt.ylabel('y')
 fig.legend(loc=2)
 plt.title(title)
 plt.savefig('{}.pdf'.format(title))
plt.show()
date = list(map(to_date, x_pred))
pred = y_pred
real = y_real
for i in range(len(pred)) :
if i < len(real):
print('{}\t{:.0f}\t{:.0f}\t{:.3f}'.format(date[i], real[i], pred[i], np.abs(pred[i]-real[i])/real[i]*100))
else:
print('{}\t-\t{:.0f}'.format(date[i], pred[i]))
return pred 
<load_from_csv>
encoded_docs = t.texts_to_sequences(test.text.tolist())
 padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post' )
train_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/train.csv')
 test_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/test.csv')
 pred_data = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/submission.csv')
 train_data = train_data.fillna(value='NULL')
 test_data = test_data.fillna(value='NULL' )<count_unique_values>
y_test_pred = model.predict_classes(padded_docs )
train_date_list = train_data.iloc[:, 5].unique()
print(len(train_date_list))
print(train_date_list)
 test_date_list = test_data.iloc[:, 5].unique()
print(len(test_date_list))
print(test_date_list)
 len(train_data.groupby(['Province/State', 'Country/Region']))
len(test_data.groupby(['Province/State', 'Country/Region'])) 
<load_from_csv>
sub = pd.DataFrame()
sub['Id'] = test_id
sub['target'] = y_test_pred
sub.to_csv('submission_2.csv',index=False )
start_date = '01/22/2020'
test_date_list = test_data.iloc[:, 5].unique()
test_data_filled = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/test.csv')
 test_data_filled = test_data_filled.fillna(value='NULL')
 test_data_filled['ConfirmedCases'] = pred_data['ConfirmedCases']
test_data_filled['Fatalities'] = pred_data['Fatalities']
for idx,(k, v)in enumerate(train_data.groupby(['Province/State', 'Country/Region'])) :
print(idx, k, end=' ')
 b_cc, b_f = 5, 3
if k[1] == 'China':
b_cc = 1
b_f = 1
if k[0] == 'Hong Kong':
b_cc = 3
elif k[1] == 'Italy':
b_cc = 10
b_f = 3
elif k[1] == 'US':
b_cc = 5
b_f = 3
elif k[1] == 'Spain':
b_cc = 4
b_f = 3
hist_num = v.loc[:,'ConfirmedCases'].tolist()
ret = func_fitting(hist_num, y_scale=max(1000, np.max(hist_num)) , b=b_cc,
start_pred=60, PN=10, MAXN=len(hist_num)+30)
 ret = list(ret)
 real_cc = np.round(np.array(ret[1]))
pred_cc = np.round(np.array(ret[5]))
for i in range(len(real_cc)) :
pred_cc[i] = real_cc[i]
pred_cc = pred_cc[50:]
print(pred_cc)
 hist_num = v.loc[:,'Fatalities'].tolist()
ret = func_fitting(hist_num, y_scale=max(1000, np.max(hist_num)) , b=b_f,
start_pred=60, PN=10, MAXN=len(hist_num)+30)
 ret = list(ret)
 real_f = np.round(np.array(ret[1]))
pred_f = np.round(np.array(ret[5]))
for i in range(len(real_f)) :
pred_f[i] = real_f[i]
pred_f = pred_f[50:]
print(pred_f)
 for i in range(14, len(pred_cc)) :
if pred_f[i] < 20 and pred_cc[i] > 200 and k[1] != 'China':
pred_f[i] = pred_cc[i] * 0.01 * np.log10(pred_cc[i])
 print(pred_f)
 print(pred_cc[-1], pred_f[-1])
 for i in range(len(pred_cc)) :
index =(test_data_filled['Province/State'] == k[0])& \ 
(test_data_filled['Country/Region'] == k[1])& \ 
(test_data_filled['Date'] == test_date_list[i])
 test_data_filled.loc[index, 'ConfirmedCases'] = pred_cc[i]
test_data_filled.loc[index, 'Fatalities'] = pred_f[i] 
<filter>
submission = test_data_filled.loc[:,['ForecastId', 'ConfirmedCases', 'Fatalities']]<save_to_csv>
encoded_docs = t.texts_to_sequences(X_train.tolist())
 padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post')
 print(padded_docs )
submission.to_csv("submission.csv", index=False)
 submission.head(500 )<feature_engineering>
vocab_size
np.log10(80000 )<load_from_csv>
def define_model(vocab_size, max_length):
model = Sequential()
model.add(Embedding(vocab_size, 100, input_length=max_length))
model.add(Conv1D(filters=32, kernel_size=8, activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
 model.add(Dense(10, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.summary()
plot_model(model, to_file='model.png', show_shapes=True)
 return model
%matplotlib inline
for dirname, _, filenames in os.walk('/kaggle/input'):
print(dirname)
 data_path = Path('/kaggle/input/covid19-global-forecasting-week-1/')
 train = pd.read_csv(data_path / 'train.csv')
 test = pd.read_csv(data_path / 'test.csv')
 data_path = Path('/kaggle/input/covid19-global-forecasting-week-2/')
 train_2 = pd.read_csv(data_path / 'train.csv')
 test_2 = pd.read_csv(data_path / 'test.csv')
<feature_engineering>
model = define_model(vocab_size, max_length)
 model.fit(padded_docs, y_train, epochs=10, verbose=2 )
<feature_engineering>
loss, accuracy = model.evaluate(padded_docs, y_train, verbose=0 )
train['country+province'] = train['Country/Region'].fillna('')+ '-' + train['Province/State'].fillna('')
 train_2['country+province'] = train_2['Country_Region'].fillna('')+ '-' + train_2['Province_State'].fillna('')
 df = train.groupby('country+province')[['Lat', 'Long']].mean()
df.loc['United Kingdom-'] = df.loc['United Kingdom-United Kingdom']
df.loc['Diamond Princess-'] = df.loc['Cruise Ship-Diamond Princess']
df.loc['Denmark-'] = df.loc['Denmark-Denmark']
df.loc['France-'] = df.loc['France-France']
df.loc['Gambia-'] = df.loc['Gambia, The-']
df.loc['Netherlands-'] = df.loc['Netherlands-Netherlands']
df.loc['Dominica-'] =(15.3, -61.383333)
 df.loc['Angola-'] =(-8.830833, 13.245)
 df.loc['Bahamas-'] =(25.066667, -77.333333)
 df.loc['Belize-'] =(17.498611, -88.188611)
 df.loc['Cabo Verde-'] =(14.916667, -23.516667)
 df.loc['Chad-'] =(12.134722, 15.055833)
 df.loc['Denmark-Greenland'] =(64.181389, -51.694167)
 df.loc['El Salvador-'] =(13.698889, -89.191389)
 df.loc['Eritrea-'] =(15.322778, 38.925)
 df.loc['Fiji-'] =(-18.166667, 178.45)
 df.loc['France-Martinique'] =(14.666667, -61)
 df.loc['France-New Caledonia'] =(-22.2758, 166.458)
 df.loc['Grenada-'] =(12.05, -61.75)
 df.loc['Guinea-Bissau-'] =(11.85, -15.566667)
 df.loc['Haiti-'] =(18.533333, -72.333333)
 df.loc['Laos-'] =(17.966667, 102.6)
 df.loc['Libya-'] =(32.887222, 13.191389)
 df.loc['Madagascar-'] =(-18.933333, 47.516667)
 df.loc['Mali-'] =(12.639167, -8.002778)
 df.loc['Mozambique-'] =(-25.966667, 32.583333)
 df.loc['Netherlands-Sint Maarten'] =(18.052778, -63.0425)
 df.loc['Nicaragua-'] =(12.136389, -86.251389)
 df.loc['Niger-'] =(13.511667, 2.125278)
 df.loc['Papua New Guinea-'] =(-9.478889, 147.149444)
 df.loc['Saint Kitts and Nevis-'] =(17.3, -62.733333)
 df.loc['Syria-'] =(33.513056, 36.291944)
 df.loc['Timor-Leste-'] =(-8.566667, 125.566667)
 df.loc['Uganda-'] =(0.313611, 32.581111)
 df.loc['Zimbabwe-'] =(-17.829167, 31.052222)
 df.loc['United Kingdom-Bermuda'] =(32.293, -64.782)
 df.loc['United Kingdom-Isle of Man'] =(54.145, -4.482)
 train_2['Lat'] = train_2['country+province'].apply(lambda x: df.loc[x, 'Lat'])
 train_2['Long'] = train_2['country+province'].apply(lambda x: df.loc[x, 'Long'])
 train_2.head()<save_to_csv>
encoded_docs = t.texts_to_sequences(X_test.tolist())
 padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post' )
train_2.to_csv("train2_latlong.csv",index=False )<feature_engineering>
y_pred = model.predict_classes(padded_docs )
test_2['country+province'] = test_2['Country_Region'].fillna('')+ '-' + test_2['Province_State'].fillna('')
 test_2['Lat'] = test_2['country+province'].apply(lambda x: df.loc[x, 'Lat'])
 test_2['Long'] = test_2['country+province'].apply(lambda x: df.loc[x, 'Long'])
 test_2.head()<save_to_csv>
encoded_docs = t.texts_to_sequences(test.text.tolist())
 padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post' )
test_2.to_csv("test2_latlong.csv",index=False )<feature_engineering>
y_test_pred = model.predict_classes(padded_docs )
<import_modules>
sub = pd.DataFrame()
sub['Id'] = test_id
sub['target'] = y_test_pred
sub.to_csv('submission_cnn.csv',index=False )
import lightgbm as lgb
from fastai.tabular import *
from sklearn import preprocessing
import datetime<load_from_csv>
model.save('model.h5' )
train = pd.read_csv("train2_latlong.csv")
 test = pd.read_csv("test2_latlong.csv")
 sub = pd.read_csv(".. /input/covid19-global-forecasting-week-2/submission.csv" )<concatenate>
def define_model(length, vocab_size):
inputs1 = Input(shape=(length,))
embedding1 = Embedding(vocab_size, 100 )(inputs1)
 conv1 = Conv1D(32, 4, activation='relu' )(embedding1)
 drop1 = Dropout(0.5 )(conv1)
 pool1 = MaxPooling1D()(drop1)
 flat1 = Flatten()(pool1)
 inputs2 = Input(shape=(length,))
embedding2 = Embedding(vocab_size, 100 )(inputs2)
 conv2 = Conv1D(32, 6, activation='relu' )(embedding2)
 drop2 = Dropout(0.5 )(conv2)
 pool2 = MaxPooling1D()(drop2)
 flat2 = Flatten()(pool2)
 inputs3 = Input(shape=(length,))
embedding3 = Embedding(vocab_size, 100 )(inputs3)
 conv3 = Conv1D(32, 8, activation='relu' )(embedding3)
 drop3 = Dropout(0.5 )(conv3)
 pool3 = MaxPooling1D()(drop3)
 flat3 = Flatten()(pool3)
 merged = concatenate([flat1, flat2, flat3])
 dense1 = Dense(10, activation='relu' )(merged)
 outputs = Dense(1, activation='sigmoid' )(dense1)
 model = Model(inputs=[inputs1, inputs2, inputs3], outputs=outputs)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 model.summary()
plot_model(model, show_shapes=True, to_file='multichannel.png')
 return model
train = train.append(test[test['Date']>'2020-03-26'] )<compute_test_metric>
encoded_docs = t.texts_to_sequences(X_train.tolist())
 padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post')
def rmsle(y_true, y_pred):
return np.sqrt(np.mean(np.power(np.log1p(y_pred)- np.log1p(y_true), 2)))
 def mape(y_true, y_pred):
return np.mean(np.abs(y_pred -y_true)*100/(y_true+1))<data_type_conversions>
model = define_model(max_length,vocab_size)
 model.fit([padded_docs,padded_docs,padded_docs], array(y_train), epochs=7, batch_size=16 )
train['Date'] = pd.to_datetime(train['Date'], format='%Y-%m-%d' )<feature_engineering>
loss, accuracy = model.evaluate([padded_docs,padded_docs,padded_docs], y_train, verbose=0 )
train['day_dist'] = train['Date']-train['Date'].min()
train['day_dist'] = train['day_dist'].dt.days
cat_cols = train.dtypes[train.dtypes=='object'].keys()
cat_cols<categorify>
encoded_docs = t.texts_to_sequences(X_test.tolist())
 padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post' )
for cat_col in cat_cols:
train[cat_col].fillna('no_value', inplace = True)
 train['place'] = train['Province_State']+'_'+train['Country_Region']
for cat_col in ['place']:
le = preprocessing.LabelEncoder()
le.fit(train[cat_col])
 train[cat_col]=le.transform(train[cat_col] )<feature_engineering>
_, acc = model.evaluate([padded_docs,padded_docs,padded_docs], array(y_test), verbose=0)
 print('Train Accuracy: %.2f' %(acc*100))
<filter>
encoded_docs = t.texts_to_sequences(test.text.tolist())
 padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post' )
val = train[(train['Date']>='2020-03-19')&(train['Id'].isnull() ==False)]<prepare_x_and_y>
y_test_pred = model.predict([padded_docs,padded_docs,padded_docs] )
y_ft = train["Fatalities"]
y_val_ft = val["Fatalities"]
y_cc = train["ConfirmedCases"]
y_val_cc = val["ConfirmedCases"]<init_hyperparams>
sub = pd.DataFrame()
sub['Id'] = test_id
sub['target'] = y_test_pred
sub.to_csv('submission_multi-cnn.csv',index=False )
params = {
"objective": "regression",
"boosting": 'gbdt',
"num_leaves": 1280,
"learning_rate": 0.05,
"feature_fraction": 0.9,
"reg_lambda": 2,
"metric": "rmse",
'min_data_in_leaf':20
}<filter>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
dates = dates[dates>'2020-03-26']
len(dates )<define_variables>
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
import tokenization
drop_cols = ['Id', 'ConfirmedCases', 'Fatalities','day_dist', 'Province_State', 'Country_Region', 'Date', 'country+province']<count_unique_values>
train= pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test=pd.read_csv('.. /input/nlp-getting-started/test.csv' )
places = train["country+province"].unique()
len(places )<define_variables>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
<feature_engineering>
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 return model
places_dict={}
for place in places:
initdate = min(train.loc[(train["country+province"]==place)&(train["ConfirmedCases"]>0)]["Date"])
 places_dict[place] = initdate.date()
places_dict<feature_engineering>
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
days_from_first_case = []
for i in range(len(train)) :
row = train.iloc[i]
row = row.to_dict()
initdate = places_dict[row["country+province"]]
currdate = row["Date"].date()
days =(currdate - initdate ).days
days_from_first_case.append(max(0,days))<feature_engineering>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
train['Days_from_first_case'] = days_from_first_case<save_to_csv>
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
train.to_csv("finaltrain.csv",index=False )<filter>
train_input = bert_encode(train.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test.text.values, tokenizer, max_len=160)
 train_labels = train.target.values
test[test['Country_Region']=='India']<load_from_csv>
train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=3,
batch_size=16
)
train_sub = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv" )<merge>
test_pred = model.predict(test_input )
<feature_engineering><EOS> 
submission=pd.DataFrame()
submission['Id']=test_id
submission['target'] = test_pred.round().astype(int)
 submission.to_csv('submission_3.csv', index=False)