Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<merge>
np.random.seed(0)
 plt.style.use('ggplot')
 stop=set(stopwords.words('english'))
np.random.seed(1)
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
eval2 = Fatalities_all_result_final[['key','Date','best_pred','best_pred_1']].merge(test, how='right', on=['key','Date'])
 eval2.rename(columns={'best_pred':'Fatalities'},inplace=True)
 eval2['Fatalities']=eval2['Fatalities'].fillna(0)
 eval2<merge>
train= pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test=pd.read_csv('.. /input/nlp-getting-started/test.csv')
 train.head()
sub_prep = eval1[['ForecastId','ConfirmedCases','key']].merge(eval2[['ForecastId','Fatalities']], on=['ForecastId'], how='left')
 sub_prep<merge>
train.isnull().sum(axis=0 )
sub = sub_prep.merge(submission['ForecastId'], on=['ForecastId'], how='right')
 sub<sort_values>
test.isnull().sum(axis=0 )
sub=sub[['ForecastId','ConfirmedCases','Fatalities']]
sub=sub.sort_values('ForecastId')
 sub<save_to_csv>
keyword_cnt = train.keyword.value_counts()
keyword_cnt
sub.to_csv('submission.csv',header=['ForecastId','ConfirmedCases','Fatalities'],index=False)
 sub<merge>
train_fake = train[train['target'] == 1]
keyword_cnt_fake = train_fake.keyword.value_counts()
keyword_cnt_fake
train['Date']=train.Date.astype('datetime64[ns]')
 verify=train[['key','Date','ConfirmedCases','Fatalities']].merge(test[['key','Date','ForecastId']], how='inner', on=['key','Date'])
 pred=verify[['ForecastId']].merge(sub, how='inner', on=['ForecastId'] )<compute_test_metric>
n_corpus=[]
for text in tqdm(train['text']):
text = re.sub(r'https?://\S+|www\.\S+', '', text)
 text = re.sub(r'<.*?>', '', text)
 text = re.sub(r'[^a-zA-Z0-9]+', ' ', text)
 text = re.sub(r'[0-9]', '', text)
 text = text.lower()
text = nltk.word_tokenize(text)
 ps = PorterStemmer()
text = [ps.stem(word)for word in text if not word in set(stopwords.words('english')) ]
text = ' '.join(text)
 n_corpus.append(text )
RMSLE(pred['Fatalities'].values,verify['Fatalities'].values )<compute_test_metric>
train['text_n']=n_corpus
train.drop('text',axis=1 )
RMSLE(pred['ConfirmedCases'].values,verify['ConfirmedCases'].values )<groupby>
n_corpus=[]
for text in tqdm(test['text']):
text = re.sub(r'https?://\S+|www\.\S+', '', text)
 text = re.sub(r'<.*?>', '', text)
 text = re.sub(r'[^a-zA-Z0-9]+', ' ', text)
 text = re.sub(r'[0-9]', '', text)
 text = text.lower()
text = nltk.word_tokenize(text)
 ps = PorterStemmer()
text = [ps.stem(word)for word in text if not word in set(stopwords.words('english')) ]
text = ' '.join(text)
 n_corpus.append(text )
train
train_ck=train.groupby(['Country/Region','Date'] ).sum().reset_index()
train_ck['key']=train_ck['Country/Region'] 
<create_dataframe>
test['text_n']=n_corpus
test.drop('text',axis=1 )
Fatalities_all_result_final=pd.DataFrame()
ConfirmedCases_all_result_Final=pd.DataFrame()
for keys in train_ck['key'].unique() :
chk=train_ck[train_ck['key']==keys]
chk.index=chk.Date
fcastperiod=0
fcastperiod1=35
actual=chk[:chk.shape[0]-fcastperiod]
ffcast=chk[chk.shape[0]-fcastperiod-1:]
ffcast
try:
Fatalities_all_result_1=pred_ets(fcastperiod,fcastperiod1,actual,ffcast,'Fatalities' ).reset_index()
except:
Fatalities_all_result_1=pd.DataFrame(pd.date_range(start=train.Date.min() , periods=60+fcastperiod1+1, freq='D')[1:])
 Fatalities_all_result_1.columns=['index']
Fatalities_all_result_1['best_model']='naive'
Fatalities_all_result_1['best_pred']=0
Fatalities_all_result_1['key']=keys
Fatalities_all_result_final=Fatalities_all_result_final.append(Fatalities_all_result_1,sort=True)
 try:
ConfirmedCases_all_result_1=pred_ets(fcastperiod,fcastperiod1,actual,ffcast,'ConfirmedCases' ).reset_index()
except:
ConfirmedCases_all_result_1=pd.DataFrame(pd.date_range(start=train.Date.min() , periods=60+fcastperiod1+1, freq='D')[1:])
 ConfirmedCases_all_result_1.columns=['index']
ConfirmedCases_all_result_1['best_model']='naive'
ConfirmedCases_all_result_1['best_pred']=1
ConfirmedCases_all_result_1['key']=keys
ConfirmedCases_all_result_Final=ConfirmedCases_all_result_Final.append(ConfirmedCases_all_result_1,sort=True)
 print(' done for %s' % keys )<feature_engineering>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
ConfirmedCases_all_result_Final.rename(columns={'index':'Date'},inplace=True)
 Fatalities_all_result_final.rename(columns={'index':'Date'},inplace=True)
 ConfirmedCases_all_result_Final['best_pred']=np.where(ConfirmedCases_all_result_Final['best_pred'] is np.nan , 0, ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred']=np.where(Fatalities_all_result_final['best_pred'] is np.nan , 0 , Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred']=np.where(ConfirmedCases_all_result_Final['best_pred'] <0 , 0, ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred']=np.where(Fatalities_all_result_final['best_pred'] <0 , 0 , Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['AdditiveDamped'] is np.nan , ConfirmedCases_all_result_Final['best_pred'] , ConfirmedCases_all_result_Final['AdditiveDamped'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['AdditiveDamped'] is np.nan , Fatalities_all_result_final['best_pred'] , Fatalities_all_result_final['AdditiveDamped'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['best_pred'] is np.nan , 0, ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['best_pred'] is np.nan , 0 , Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['best_pred'] <0 , 0, ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['best_pred'] <0 , 0 , Fatalities_all_result_final['best_pred'] )<remove_duplicates>
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow_hub as hub
import tokenization
best_model_key=ConfirmedCases_all_result_Final[['key','best_model']].drop_duplicates()
max_number_current=train_ck.groupby('key' ).max() [['ConfirmedCases','Fatalities']].reset_index()
best_model_key=best_model_key.merge(max_number_current,on='key',how='left')
 best_model_key=best_model_key.sort_values('ConfirmedCases',ascending=False ).reset_index(drop=True )<load_from_csv>
def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)
 tokens += [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments )
train = pd.read_csv('.. /input/covid19-global-forecasting-week-1/train.csv')
 test = pd.read_csv('.. /input/covid19-global-forecasting-week-1/test.csv' )<feature_engineering>
def build_model(bert_layer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(clf_output)
 model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 return model
train['date_datetime'] = train['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')) )<data_type_conversions>
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
def days_convert(date):
return(date - datetime.datetime(2020,1,22)).days
train['days_since_start'] = train['date_datetime'].apply(days_convert )<feature_engineering>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case )
def rename_countries(country):
if country == 'US':
country = 'United States'
elif country == 'Gambia, The' or country == 'The Gambia':
country = "Gambia"
elif country == 'The Bahamas':
country = 'Bahamas'
elif country == 'Taiwan*':
country = 'Taiwan'
elif country == 'Republic of the Congo' or country == 'Congo(Kinshasa)' or country == 'Congo(Brazzaville)':
country = 'DR Congo'
elif country == 'Korea, South':
country = 'South Korea'
elif country == 'Czechia':
country = 'Czech Republic'
return country
train['Country/Region'] = train['Country/Region'].apply(rename_countries )<feature_engineering>
train_input = bert_encode(train.text.values, tokenizer, max_len=160)
 test_input = bert_encode(test.text.values, tokenizer, max_len=160)
 train_labels = train.target.values
def clean_name(country):
country = country.split('[')[0]
return country
population['country'] = population['Country or area'].apply(clean_name)
 population['pop'] = population['Population(1 July 2019)']<filter>
train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=3,
batch_size=16 
)
 model.save('model.h5' )
<feature_engineering><EOS> 
test_pred = model.predict(test_input)
 submission=pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 submission['target'] = test_pred.round().astype(int)
 submission.to_csv('submission.csv', index=False)