Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<feature_engineering>
np.random.seed(1)
 nltk.download('stopwords')
 tf.random.set_seed(1)
 pd.set_option('display.max_colwidth', 500)
 warnings.filterwarnings('ignore' )
test.loc[test['Fatalities_x'].isnull() ==True, 'Fatalities_x'] = test.loc[test['Fatalities_x'].isnull() ==True, 'Fatalities_y']<filter>
train = pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test = pd.read_csv(".. /input/nlp-getting-started/test.csv")
 print("Train Shape :", train.shape)
 print("Test Shape :", test.shape )
last_amount = test.loc[(test['Country_Region']=='Italy')&(test['Date']=='2020-03-26'),'ConfirmedCases_x']<filter>
train.isnull().sum()
last_fat = test.loc[(test['Country_Region']=='Italy')&(test['Date']=='2020-03-24'),'Fatalities_x']<feature_engineering>
train.isnull().sum()
i, k = 0, 35
for date in dates:
k = k-1
i = i + 1
test.loc[(test['Country_Region']=='Italy')&(test['Date']==date),'ConfirmedCases_x'] = last_amount.values[0]+i*(5000-(100*i))
test.loc[(test['Country_Region']=='Italy')&(test['Date']==date),'Fatalities_x'] = last_fat.values[0]+i*(800-(10*i))<filter>
train['target'].value_counts(normalize = True )
test.loc[(test['Country_Region']=='Italy')]<rename_columns>
train.keyword.value_counts()
sub = test[['ForecastId', 'ConfirmedCases_x','Fatalities_x']]
sub.columns = ['ForecastId', 'ConfirmedCases', 'Fatalities']<feature_engineering>
train.location.value_counts()
sub.loc[sub['ConfirmedCases']<0, 'ConfirmedCases'] = 0<feature_engineering>
real_tweets = train[train['target']==1]['text']
real_tweets.values[0:5]
sub.loc[sub['Fatalities']<0, 'Fatalities'] = 0<save_to_csv>
fake_tweets = train[train['target']==0]['text']
fake_tweets.values[0:5]
sub.to_csv('submission.csv',index=False )<load_from_csv>
def clean_text(text):
text = text.lower()
text = re.sub('\[.*?\]', '', text)
 text = re.sub('https?://\S+|www\.\S+', '', text)
 text = re.sub('<.*?>+', '', text)
 text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
 text = re.sub('
', '', text)
 text = re.sub('\w*\d\w*', '', text)
 return text
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/train.csv')
 test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/test.csv')
 submission = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-1/submission.csv' )<data_type_conversions>
train['cleaned_text'] = train['text'].apply(lambda x: clean_text(x))
test['cleaned_text'] = test['text'].apply(lambda x: clean_text(x))
train['cleaned_text'].head()
test['Date']=test.Date.astype('datetime64[ns]' )<data_type_conversions>
!pip install nlppreprocess
nlp = NLP()
train['stopwords_cleaned'] = train['cleaned_text'].apply(nlp.process)
 test['stopwords_cleaned'] = test['cleaned_text'].apply(nlp.process )
train['key']=train['Province/State'].astype('str')+ " " + train['Country/Region'].astype('str')+ " " +train['Lat'].astype('str')+ " " +train['Long'].astype('str' )<data_type_conversions>
en_model = spacy.load('en', disable=['parser', 'ner'])
 def lemmatization(texts):
output = []
for i in texts:
s = [token.lemma_ for token in en_model(i)]
output.append(' '.join(s))
return output
test['key']=test['Province/State'].astype('str')+ " " + test['Country/Region'].astype('str')+ " " +test['Lat'].astype('str')+ " " +test['Long'].astype('str' )<groupby>
train['lemmatized_text'] = lemmatization(train['stopwords_cleaned'])
 test['lemmatized_text'] = lemmatization(test['stopwords_cleaned'] )
daily_analysis=train.groupby(['Date'] ).sum()<compute_test_metric>
x_train, x_test, y_train, y_test = train_test_split(train['stopwords_cleaned'], train['target'],
test_size = 0.2, random_state = 1 )
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)) )<feature_engineering>
hub_layer = hub.KerasLayer('https://tfhub.dev/google/universal-sentence-encoder/4',
input_shape = [],
output_shape = [512],
dtype = tf.string,
trainable = True)
 model = tf.keras.models.Sequential()
model.add(hub_layer)
 model.add(tf.keras.layers.Dense(128, activation = 'relu'))
model.add(tf.keras.layers.Dense(32, activation = 'relu'))
model.add(tf.keras.layers.Dense(1, activation = 'sigmoid'))
model.summary()
train_lag_1=train.groupby(['key'] ).shift(periods=1)
 train_lag_2=train.groupby(['key'] ).shift(periods=2)
 train_lag_3=train.groupby(['key'] ).shift(periods=3)
 train_lag_4=train.groupby(['key'] ).shift(periods=4)
 train['lag_1_ConfirmedCases']=train_lag_1['ConfirmedCases']
train['lag_1_Fatalities']=train_lag_1['Fatalities']
train['lag_2_ConfirmedCases']=train_lag_2['ConfirmedCases']
train['lag_2_Fatalities']=train_lag_2['Fatalities']
train['lag_3_ConfirmedCases']=train_lag_3['ConfirmedCases']
train['lag_3_Fatalities']=train_lag_3['Fatalities']
train['lag_4_ConfirmedCases']=train_lag_4['ConfirmedCases']
train['lag_4_Fatalities']=train_lag_4['Fatalities']
train<train_model>
model.compile(optimizer = 'adam',
loss = 'binary_crossentropy',
metrics = ['accuracy'] )
def pred_ets(fcastperiod,fcastperiod1,actual,ffcast,type_ck='ConfirmedCases',verbose=False):
actual=actual[actual[type_ck]>0]
index=pd.date_range(start=ffcast.index[0], end=ffcast.index[-1], freq='D')
 data=ffcast[type_ck].values
ffcast1 = pd.Series(data, index)
 index=pd.date_range(start=actual.index[0], end=actual.index[-1], freq='D')
 data=actual[type_ck].values
daily_analysis_dat = pd.Series(data, index)
 livestock2=daily_analysis_dat
fit=[]
fcast=[]
fname=[]
try:
fit1 = SimpleExpSmoothing(livestock2 ).fit()
fcast1 = fit1.forecast(fcastperiod1 ).rename("SES")
 fit.append(fit1)
 fcast.append(fcast1)
 fname.append('SES')
 except:
1==1
try:
fit2 = Holt(livestock2 ).fit()
fcast2 = fit2.forecast(fcastperiod1 ).rename("Holt")
 fit.append(fit2)
 fcast.append(fcast2)
 fname.append('Holt')
 except:
1==1
try:
fit3 = Holt(livestock2, exponential=True ).fit()
fcast3 = fit3.forecast(fcastperiod1 ).rename("Exponential")
 fit.append(fit3)
 fcast.append(fcast3)
 fname.append('Exponential')
 except:
1==1
try:
fit4 = Holt(livestock2, damped=True ).fit(damping_slope=0.98)
 fcast4 = fit4.forecast(fcastperiod1 ).rename("AdditiveDamped")
 fit.append(fit4)
 fcast.append(fcast4)
 fname.append('AdditiveDamped')
 except:
1==1
try:
fit5 = Holt(livestock2, exponential=True, damped=True ).fit()
fcast5 = fit5.forecast(fcastperiod1 ).rename("MultiplicativeDamped")
 fit.append(fit5)
 fcast.append(fcast5)
 fname.append('MultiplicativeDamped')
 except:
1==1
try:
fit6 = Holt(livestock2, damped=True ).fit()
fcast6 = fit6.forecast(fcastperiod1 ).rename("AdditiveDampedC")
 fit.append(fit6)
 fcast.append(fcast6)
 fname.append('AdditiveDampedC')
 except:
1==1
def RMSLE(pred,actual):
return np.sqrt(np.mean(np.power(( np.log(pred+1)-np.log(actual+1)) ,2)))
 pred_all_result=pd.concat([pd.DataFrame(k.fittedvalues)for k in fit],axis=1)
 pred_all_result.columns=fname
all_result=pd.concat([pd.DataFrame(k)for k in fcast],axis=1)
 col_chk=[]
vvvl=ffcast[type_ck].values.shape[0]
for k in all_result.columns:
if verbose: print("actual value for method %s is = %s" %(k,RMSLE(all_result[k].values,ffcast[type_ck].values)))
 if RMSLE(all_result[k].values[:vvvl],ffcast[type_ck].values)is not np.nan:
col_chk.append(k)
 col_chk_f=[]
min_acc=-1
for k in col_chk:
acc=RMSLE(pred_all_result[k].values,actual[type_ck].values)
 if verbose: print("pred value for method %s is = %s" %(k,acc))
if acc is not np.nan:
col_chk_f.append(k)
 if min_acc==-1:
min_acc=acc
model_select=k
elif acc<min_acc:
min_acc=acc
model_select=k
all_result=all_result.append(pred_all_result,sort=False)
 all_result['best_model']=model_select
all_result['best_pred']=all_result[model_select]
return all_result 
<create_dataframe>
model.fit(x_train,
y_train,
epochs = 1,
validation_data =(x_test, y_test))
Fatalities_all_result_final=pd.DataFrame()
ConfirmedCases_all_result_Final=pd.DataFrame()
for keys in train['key'].unique() :
chk=train[train['key']==keys]
chk.index=chk.Date
fcastperiod=0
fcastperiod1=35
actual=chk[:chk.shape[0]-fcastperiod]
ffcast=chk[chk.shape[0]-fcastperiod-1:]
ffcast
try:
Fatalities_all_result_1=pred_ets(fcastperiod,fcastperiod1,actual,ffcast,'Fatalities' ).reset_index()
except:
Fatalities_all_result_1=pd.DataFrame(pd.date_range(start=chk.Date.min() , periods=60+fcastperiod1+1, freq='D')[1:])
 Fatalities_all_result_1.columns=['index']
Fatalities_all_result_1['best_model']='naive'
Fatalities_all_result_1['best_pred']=0
Fatalities_all_result_1['key']=keys
Fatalities_all_result_final=Fatalities_all_result_final.append(Fatalities_all_result_1,sort=True)
 try:
ConfirmedCases_all_result_1=pred_ets(fcastperiod,fcastperiod1,actual,ffcast,'ConfirmedCases' ).reset_index()
except:
ConfirmedCases_all_result_1=pd.DataFrame(pd.date_range(start=train.Date.min() , periods=60+fcastperiod1+1, freq='D')[1:])
 ConfirmedCases_all_result_1.columns=['index']
ConfirmedCases_all_result_1['best_model']='naive'
ConfirmedCases_all_result_1['best_pred']=1
ConfirmedCases_all_result_1['key']=keys
ConfirmedCases_all_result_Final=ConfirmedCases_all_result_Final.append(ConfirmedCases_all_result_1,sort=True)
 print(' done for %s' % keys)
 sys.stdout = orig_stdout<feature_engineering>
pred = model.predict_classes(test['stopwords_cleaned'] )
ConfirmedCases_all_result_Final.rename(columns={'index':'Date'},inplace=True)
 Fatalities_all_result_final.rename(columns={'index':'Date'},inplace=True)
 ConfirmedCases_all_result_Final['best_pred']=np.where(ConfirmedCases_all_result_Final['best_pred'] is np.nan , 0, ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred']=np.where(Fatalities_all_result_final['best_pred'] is np.nan , 0 , Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred']=np.where(ConfirmedCases_all_result_Final['best_pred'] <0 , 0, ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred']=np.where(Fatalities_all_result_final['best_pred'] <0 , 0 , Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['AdditiveDamped'] is np.nan , ConfirmedCases_all_result_Final['best_pred'] , ConfirmedCases_all_result_Final['AdditiveDamped'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['AdditiveDamped'] is np.nan , Fatalities_all_result_final['best_pred'] , Fatalities_all_result_final['AdditiveDamped'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['best_pred'] is np.nan , 0, ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['best_pred'] is np.nan , 0 , Fatalities_all_result_final['best_pred'])
 ConfirmedCases_all_result_Final['best_pred_1']=np.where(ConfirmedCases_all_result_Final['best_pred'] <0 , 0, ConfirmedCases_all_result_Final['best_pred'])
 Fatalities_all_result_final['best_pred_1']=np.where(Fatalities_all_result_final['best_pred'] <0 , 0 , Fatalities_all_result_final['best_pred'])
<data_type_conversions>
submission = pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv")
 submission['target'] = pred
submission.to_csv('submission.csv', index=False )
<merge><EOS> 
submission.to_csv("submission.csv", index = False )