Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_from_csv>
!pip install bert-for-tf2
!pip install sentencepiece
life = pd.read_csv('/kaggle/input/world-bank-data-1960-to-2016/life_expectancy.csv')[['Country Name','2016']]
life.head()<feature_engineering>
try:
%tensorflow_version 2.x
except Exception:
pass
def change_countries(country):
if country == 'DR Congo':
country = 'Democratic Republic of the Congo'
elif country == 'Bahamas':
country = 'The Bahamas'
return country
def get_expectancy(country):
try:
return life[life['Country Name'] == country]['2016'].reset_index() ['2016'][0]
except:
if country == 'Brunei':
return 77
elif country == 'Democratic Republic of the Congo':
return 60
elif country == 'Cruise Ship':
return 85
elif country == 'Egypt':
return 72.7
elif country == 'Eswatini':
return 42
elif country == 'French Guiana':
return 75.9
elif country == 'Gambia':
return 64.9
elif country == 'Guadeloupe':
return 81.84
elif country == 'Guernsey':
return 82.5
elif country == 'Holy See':
return 78
elif country == 'Iran':
return 71.4
elif country == 'Jersey':
return 81.9
elif country == 'South Korea':
return 82.7
elif country == 'Kyrgyzstan':
return 71.4
elif country == 'Martinique':
return 79.6
elif country == 'Mayotte':
return 76.83
elif country == 'North Macedonia':
return 76
elif country == 'Reunion':
return 76.4
elif country == 'Russia':
return 70.8
elif country == 'Saint Lucia':
return 77.8
elif country == 'Saint Vincent and the Grenadines':
return 75.3
elif country == 'Slovakia':
return 77.1
elif country == 'Taiwan':
return 80.2
elif country == 'The Bahamas':
return 72.4
elif country == 'Gambia':
return 64.9
elif country == 'Venezuela':
return 75.8
print(country)
 train['life_expectancy'] = train['Country/Region2'].apply(get_expectancy )<feature_engineering>
def recall_m(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision_m(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
def f1_m(y_true, y_pred):
precision = precision_m(y_true, y_pred)
 recall = recall_m(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
for index in train[train['Country/Region']=='Andorra']['life_expectancy'].index:
train.loc[index,'life_expectancy'] = 82.8
for index in train[train['Country/Region']=='Greenland']['life_expectancy'].index:
train.loc[index,'life_expectancy'] = 72.4
for index in train[train['Country/Region']=='Monaco']['life_expectancy'].index:
train.loc[index,'life_expectancy'] = 89.5
for index in train[train['Country/Region']=='San Marino']['life_expectancy'].index:
train.loc[index,'life_expectancy'] = 85.6 
<filter>
train= pd.read_csv('.. /input/extensive-pre-processing-for-bert/processed train.csv')
 train.head(5 )
train[train['life_expectancy'].isna() ==True]<split>
train.loc[4,'processed_text']
X = train[['Lat','Long','days_since_start','population','median_age']]
y = train['ConfirmedCases']
X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.3 )<train_model>
test=pd.read_csv('.. /input/extensive-pre-processing-for-bert/processed test.csv')
 test = test.set_index(test['id'])
 test.head(5 )
rf = RandomForestClassifier()
rf.fit(X_train,y_train )<compute_train_metric>
test_actual = pd.read_csv("https://raw.githubusercontent.com/sampath9dasari/GSU/master/true%20submission.csv")
 test_labels = test_actual.target.to_numpy()
mean_absolute_error(rf.predict(X_test),y_test )<split>
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True)
X1 = train[['Lat','Long','days_since_start','population','median_age','life_expectancy']]
y1 = train['Fatalities']
X_train1,X_test1,y_train1,y_test1 = train_test_split(X1,y1,test_size=0.3 )<train_model>

 def bert_encode(texts, tokenizer, max_len=50):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments)
rf1 = RandomForestClassifier()
rf1.fit(X_train1,y_train1 )<compute_train_metric>
BertTokenizer = bert.bert_tokenization.FullTokenizer
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = BertTokenizer(vocab_file, do_lower_case )
mean_absolute_error(rf1.predict(X_test1),y_test1 )<load_from_csv>
full_input = bert_encode(train.processed_text.values, tokenizer, max_len=50)
 full_labels = train.target.values.copy()
test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/test.csv')
 test.head()<feature_engineering>
train_data, val_data, train_labels, val_labels = train_test_split(train.processed_text.values, train.target.values, test_size=0.15, random_state=10)
 train_input = bert_encode(train_data, tokenizer, max_len=50)
 val_input = bert_encode(val_data, tokenizer, max_len=50)
 test_input = bert_encode(test.processed_text.values, tokenizer, max_len=50 )
test['date_datetime'] = test['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 def days_convert(date):
return(date - datetime.datetime(2020,1,22)).days
test['days_since_start'] = test['date_datetime'].apply(days_convert)
 test['Country/Region'] = test['Country/Region'].apply(rename_countries)
 test['population'] = test['Country/Region'].apply(get_population)
 def change_countries(country):
if country == 'DR Congo':
country = 'Democratic Republic of the Congo'
elif country == 'Bahamas':
country = 'The Bahamas'
return country
test['Country/Region2'] = test['Country/Region'].apply(change_countries)
 test['median_age'] = test['Country/Region2'].apply(get_median)
 test['life_expectancy'] = test['Country/Region2'].apply(get_expectancy)
 for index in test[test['Country/Region']=='Andorra']['life_expectancy'].index:
test.loc[index,'life_expectancy'] = 82.8
for index in test[test['Country/Region']=='Greenland']['life_expectancy'].index:
test.loc[index,'life_expectancy'] = 72.4
for index in test[test['Country/Region']=='Monaco']['life_expectancy'].index:
test.loc[index,'life_expectancy'] = 89.5
for index in test[test['Country/Region']=='San Marino']['life_expectancy'].index:
test.loc[index,'life_expectancy'] = 85.6
test.head()<predict_on_test>
predictions = rf.predict(test[['Lat','Long','days_since_start','population','median_age']])
 predictions1 = rf1.predict(test[['Lat','Long','days_since_start','population','median_age','life_expectancy']] )<load_from_csv>
learning_rate=1e-5
decay=5e-5
max_len=50
lr_schedule = [9e-7,1e-8,5e-8,9e-8,7e-9,1e-9]
K.clear_session()
submit = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/submission.csv')
 submit['ConfirmedCases'] = predictions
submit['ConfirmedCases'] = submit['ConfirmedCases'].apply(int)
 submit['Fatalities'] = predictions1
submit['Fatalities'] = submit['Fatalities'].apply(int)
 submit.head()<save_to_csv>
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1024, activation='relu', activity_regularizer=regularizers.l2(1e-5))(clf_output)
 out = Dense(1024, activation='relu', activity_regularizer=regularizers.l2(1e-5))(out)
 out = Dense(1024, activation='relu', activity_regularizer=regularizers.l2(1e-5))(out)
 out = Dense(1, activation='sigmoid' )(out)
submit.to_csv('submission.csv',index=False )<save_to_csv>
sBERT = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 sBERT.compile(Adam(lr=learning_rate, decay=decay), loss='binary_crossentropy', metrics=['accuracy',f1_m])
 sBERT.summary()
test.to_csv('test.csv')
 train.to_csv('train.csv' )<install_modules>
init_weights = sBERT.get_weights()
!pip install pyramid-arima<set_options>
checkpoint1 = ModelCheckpoint('best_accuracy.h5',
monitor='val_f1_m',
save_best_only=True)
 train_history = sBERT.fit(
 full_input, full_labels,
epochs = 1,
batch_size = 16 
)
 test_pred = sBERT.predict(test_input)
 print(" - test_f1_score: {}".format(f1_score(test_labels,test_pred.round())))
 print()
sBERT.save_weights('best_accuracy.h5')
 test_accuracy = f1_score(test_labels,test_pred.round() )
warnings.filterwarnings("ignore")
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-1/test.csv")
 train = pd.read_csv(".. /input/covid19-global-forecasting-week-1/train.csv")
 train.head()<count_unique_values>
K.set_value(sBERT.optimizer.lr, 1e-6)
 sBERT.fit(
 full_input, full_labels,
epochs = 1,
batch_size = 16 
)
 test_pred = sBERT.predict(test_input)
 epoch_test_accuracy = f1_score(test_labels,test_pred.round())
 print(" - test_f1_score: {}".format(epoch_test_accuracy))
print()
if epoch_test_accuracy >= test_accuracy:
sBERT.save('best_accuracy.h5')
 test_accuracy = epoch_test_accuracy
no_countr = train['Country/Region'].nunique()
no_province = train['Province/State'].nunique()
no_countr_with_prov = len(train[train['Province/State'].isna() ==False]['Country/Region'].unique())
 total_forecasting_number = no_province + no_countr - no_countr_with_prov+2
no_days = train['Date'].nunique()
print('there are ', no_countr, 'unique Countries/Regionions, each with ', no_days, 'days of data, all of them having the same dates.There are also ',no_province, 'Provinces/States which can be found on ', no_countr_with_prov, 'countries/ regions.' )<prepare_x_and_y>
K.set_value(sBERT.optimizer.lr, 1e-7)
 sBERT.fit(
 full_input, full_labels,
epochs = 1,
batch_size = 16 
)
 test_pred = sBERT.predict(test_input)
 epoch_test_accuracy = f1_score(test_labels,test_pred.round())
 print(" - test_f1_score: {}".format(epoch_test_accuracy))
print()
if epoch_test_accuracy >= test_accuracy:
sBERT.save('best_accuracy.h5')
 test_accuracy = epoch_test_accuracy
df = confirmed_total_date.copy()
df = pd.DataFrame({'date': [df.index[i] for i in range(len(df)) ] , 'cases': df['ConfirmedCases'].values.reshape(1,-1)[0].tolist() })
 dfog = df.copy()
def l_regr(x,y):
model = LinearRegression().fit(x, y)
 return model
x = df['cases']
x = x.drop(x.index[-1] ).values.reshape(( -1, 1))
y = df['cases']
y = y.drop(y.index[0])
 ex_slope = l_regr(x,y ).coef_
d = 0
for i in range(1,5):
plt.plot(df['cases'])
 plt.show()
plt.close()
df['prev_cases'] = df['cases'].shift(1)
 df['cases'] =(df['cases'] - df['prev_cases'])
 df = df.drop(['prev_cases'],axis=1)
 df = df.drop(df.index[0])
 x = df['cases']
x = x.drop(y.index[-1] ).values.reshape(( -1, 1))
y = df['cases']
y = y.drop(y.index[0])
 model = l_regr(x,y)
 if(abs(model.coef_)> ex_slope):
print('this is it! ', ex_slope)
 break
d += 1
ex_slope = model.coef_
print(model.coef_ )<create_dataframe>
sBERT.layers[3].trainable = False
sBERT.compile(Adam(lr=1e-6, decay=1e-6), loss='binary_crossentropy', metrics=['accuracy',f1_m] )
index = 1
cases_pred= []
fatalities_pred = []
pbar = tqdm(total=total_forecasting_number)
 while index < total_forecasting_number+1:
x = train['ConfirmedCases'].iloc[[i for i in range(no_days*(index-1),no_days*index)]].values
z = train['Fatalities'].iloc[[i for i in range(no_days*(index-1),no_days*index)]].values
index += 1
no_nul_cases = pd.DataFrame(x)
 no_nul_cases = no_nul_cases[no_nul_cases.values != 0]
if(not no_nul_cases.empty):
X = [xi for xi in no_nul_cases.values]
try:
model = pyramid.arima.auto_arima(X,seasonal=True, m=12)
 pred = model.predict(31)
 pred = pred.astype(int)
 pred = pred.tolist()
except:
model = l_regr(np.array([i for i in range(len(X)) ] ).reshape(-1, 1),X)
 pred = [(model.coef_*(len(X)+i)+ model.intercept_ ).astype('int')[0][0] for i in range(1,32)]
else:
pred = [0] * 31
pred = x[-12:].astype(int ).tolist() + pred
cases_pred+=pred
no_nul_fatalities = pd.DataFrame(z)
 no_nul_fatalities = no_nul_fatalities[no_nul_fatalities.values != 0]
if(not no_nul_fatalities.empty):
Z = [zi for zi in no_nul_fatalities.values]
try:
model = pyramid.arima.auto_arima(Z, seasonal=False, m=12)
 pred = model.predict(31)
 pred = pred.astype(int)
 pred = pred.tolist()
except:
model = l_regr(np.array([i for i in range(len(Z)) ] ).reshape(-1, 1),Z)
 pred = [(model.coef_*(len(Z)+i)+ model.intercept_ ).astype('int')[0][0] for i in range(1,32)]
else:
pred = [0] * 31
pred = z[-12:].astype(int ).tolist() + pred
fatalities_pred+=pred
pbar.update(1)
 pbar.close()<save_to_csv>
sBERT.fit(
 full_input, full_labels,
validation_data=(test_input, test_labels),
epochs = 10,
batch_size = 16 
)
 test_pred = sBERT.predict(test_input)
 epoch_test_accuracy = f1_score(test_labels,test_pred.round())
 print(" - test_f1_score: {}".format(epoch_test_accuracy))
print()
if epoch_test_accuracy >= test_accuracy:
sBERT.save('best_accuracy.h5')
 test_accuracy = epoch_test_accuracy
submission = pd.DataFrame({'ForecastId': [i for i in range(1,len(cases_pred)+1)] ,'ConfirmedCases': cases_pred, 'Fatalities': fatalities_pred})
 filename = 'submission.csv'
submission.to_csv(filename,index=False )<load_from_csv>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
train=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/train.csv',sep=',')
 test=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/test.csv',sep=',')
 submission=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/submission.csv',sep=',')
 print(train.head(2))
print(test.head(2))<train_model>
sBERT.load_weights('best_accuracy.h5' )
print("Training data:
", train.count() , "
Test data:
", test.count())
 print("
Training Missing data:
", train.isnull().sum() , "
Test Missing data:
", test.isnull().sum() )<drop_column>
bert_encoder = Model(sBERT.inputs, sBERT.layers[-5].output)
 bert_encoder.summary()
train = train.drop(['Province/State'],axis=1)
 test = test.drop(['Province/State'],axis=1)
 train.dtypes<feature_engineering>
bert_encoder.layers[3].trainable
def create_time_features(df):

 df['date'] = df.index
df['dayofweek'] = df['date'].dt.dayofweek
df['quarter'] = df['date'].dt.quarter
df['month'] = df['date'].dt.month
df['year'] = df['date'].dt.year
df['dayofyear'] = df['date'].dt.dayofyear
df['dayofmonth'] = df['date'].dt.day
df['weekofyear'] = df['date'].dt.weekofyear
X = df[['dayofweek','quarter','month','year',
'dayofyear','dayofmonth','weekofyear']]
return X<data_type_conversions>
%%time
train_embed = bert_encoder.predict(train_input)
 test_embed = bert_encoder.predict(test_input )
train["iDate"] = train["Date"].apply(lambda x: x.replace("-",""))
train["iDate"] = train["iDate"].astype(int)
 test["iDate"] = test["Date"].apply(lambda x: x.replace("-",""))
test["iDate"] = test["iDate"].astype(int)
 train['Date'] = pd.to_datetime(train['Date'])
 test['Date']= pd.to_datetime(test['Date'])
 train = train.set_index(['Date'])
 test = test.set_index(['Date'] )<feature_engineering>
with open('Train BERT 1024d Embed', 'ab')as fo:
pickle.dump(train_embed, fo)
 with open('Test BERT 1024d Embed', 'ab')as fo:
pickle.dump(test_embed, fo )
create_time_features(train)
 create_time_features(test )<feature_engineering>
from sklearn.model_selection import StratifiedKFold, KFold, GridSearchCV
from sklearn.svm import SVC
train['ConfirmedCases_today'] = train.groupby(['Country/Region'])['ConfirmedCases'].diff(1)
 train['Fatalities_today'] = train.groupby(['Country/Region'])['Fatalities'].diff(1 )<data_type_conversions>
%%time
svc_model = SVC(gamma='scale', kernel='rbf', C=3)
 svc_model.fit(train_embed, train_labels )
train['ConfirmedCases_today'] = train['ConfirmedCases_today'].fillna(0)
 train['Fatalities_today'] = train['Fatalities_today'].fillna(0)
 print(train.groupby(['Country/Region'])['ConfirmedCases_today'].sum())
 print(train.groupby(['Country/Region'])['Fatalities_today'].sum())
<count_missing_values>
import xgboost as xgb
train.isnull().sum()<count_missing_values>
%%time
clf = xgb.XGBClassifier(max_depth=200, n_estimators=400, subsample=1, learning_rate=0.07, reg_lambda=0.1, reg_alpha=0.1,\
gamma=1)
 clf.fit(train_embed, train_labels)
 predictions = clf.predict(train_embed)
 print("Training set f1_score :", np.round(f1_score(train_labels, predictions),5))
train.isnull().sum()<define_variables>
test_pred1 = clf.predict(test_embed ).round().astype(int)
 test_pred2 = svc_model.predict(test_embed ).round().astype(int)
 test_pred3 = sBERT.predict(test_input ).round().astype(int)
 print("XGBOOST: ", accuracy_score(test_labels, test_pred1), f1_score(test_labels, test_pred1))
print("SVC: ",accuracy_score(test_labels, test_pred2), f1_score(test_labels, test_pred2))
print("BERT: ",accuracy_score(test_labels, test_pred3), f1_score(test_labels, test_pred3))
input_cols = ["Lat","Long", "iDate"]
output_cols = ["ConfirmedCases","Fatalities"]
ids = submission["ForecastId"]
extra_feat = ["ConfirmedCases_today", "Fatalities_today"]<prepare_x_and_y>
sub = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 sub['target'] = test_pred1
sub.to_csv('submission_xgboost.csv', index=False)
 sub['target'] = test_pred2
sub.to_csv('submission_svc.csv', index=False)
 sub['target'] = test_pred3
sub.to_csv('submission_bertnn.csv', index=False )