Kaggle competition: Natural Language Processing with Disaster Tweets
display("Running Optimization for {} Trials.".format(MAX_EVALS))
best = fmin(fn = objective, space = space, algo = tpe.suggest, trials = trials,
max_evals = MAX_EVALS )<save_to_csv>
train_df=pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test_df=pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 def data_info(d):
print('number of variables: ',d.shape[1])
 print('number of tweets: ',d.shape[0])
 print('variables names: ')
 print(d.columns)
 print('variables data-types: ')
 print(d.dtypes)
 print('missing values: ')
 c=d.isnull().sum()
print(c[c>0] )
with open('trials.json', 'w')as f:
f.write(json.dumps(str(trials)) )<find_best_params>
train_df[train_df['target']==1].loc[:4,'text']
best_hyp = ast.literal_eval(results.loc[0, 'hyperparameters'])
 best_hyp<create_dataframe>
train_disasters={'earthquake':0,'fire':0,'fires':0,'shelter':0}
test_disasters={'earthquake':0,'fire':0,'fires':0,'shelter':0}
for i,j in zip(train_df['text'],test_df['text']):
if 'earthquake' in i.split() :
train_disasters['earthquake']+=1
elif 'earthquake' in j.split() :
test_disasters['earthquake']+=1
if 'fire' in i.split() :
train_disasters['fire']+=1
elif 'fire' in j.split() :
test_disasters['fire']+=1
if 'fires' in i.split() :
train_disasters['fires']+=1
elif 'fires' in j.split() :
test_disasters['fires']+=1
if 'shelter' in i.split() :
train_disasters['shelter']+=1
elif 'shelter' in j.split() :
test_disasters['shelter']+=1
submission, gbm_fi, valid_scores = model_gbm(train_selected, train_labels,
test_selected, test_ids,
nfolds = 10, return_preds=False)
 model_results = model_results.append(pd.DataFrame({'model': ["GBM_OPT_10Fold_SEL"],
'cv_mean': [valid_scores.mean() ],
'cv_std': [valid_scores.std() ]}),
sort = True ).sort_values('cv_mean', ascending = False )<create_dataframe>
print('number of tweets that fires,fire and earthquacke mentioned in train data: ',train_disasters)
 print('number of tweets that fires,fire and earthquacke mentioned in test data: ',test_disasters )
submission, gbm_fi, valid_scores = model_gbm(train_set, train_labels,
test_set, test_ids,
nfolds = 10, return_preds=False)
 model_results = model_results.append(pd.DataFrame({'model': ["GBM_OPT_10Fold"],
'cv_mean': [valid_scores.mean() ],
'cv_std': [valid_scores.std() ]}),
sort = True ).sort_values('cv_mean', ascending = False )<train_model>
def tokenization(text):
lst=text.split()
return lst
train_df['text']=train_df['text'].apply(tokenization)
 test_df['text']=test_df['text'].apply(tokenization )
submission, gbm_fi, valid_scores = model_gbm(train_selected, train_labels,
test_selected, test_ids,
nfolds = 10, return_preds=False )<save_to_csv>
def lowercasing(lst):
new_lst=[]
for i in lst:
i=i.lower()
new_lst.append(i)
 return new_lst
train_df['text']=train_df['text'].apply(lowercasing)
 test_df['text']=test_df['text'].apply(lowercasing )
submission.to_csv('gbm_opt_10fold_selected.csv', index = False )<count_values>
def remove_punctuations(lst):
new_lst=[]
for i in lst:
for j in s.punctuation:
i=i.replace(j,'')
 new_lst.append(i)
 return new_lst
train_df['text']=train_df['text'].apply(remove_punctuations)
 test_df['text']=test_df['text'].apply(remove_punctuations )
heads['Target'].value_counts()<count_values>
def remove_numbers(lst):
nodig_lst=[]
new_lst=[]
for i in lst:
for j in s.digits:
i=i.replace(j,'')
 nodig_lst.append(i)
 for i in nodig_lst:
if i!='':
new_lst.append(i)
 return new_lst
train_df['text']=train_df['text'].apply(remove_numbers)
 test_df['text']=test_df['text'].apply(remove_numbers )
preds['Target'].value_counts()<import_modules>
def remove_stopwords(lst):
stop=stopwords.words('english')
 new_lst=[]
for i in lst:
if i not in stop:
new_lst.append(i)
 return new_lst
train_df['text']=train_df['text'].apply(remove_stopwords)
 test_df['text']=test_df['text'].apply(remove_stopwords )
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns<import_modules>
def remove_spaces(lst):
new_lst=[]
for i in lst:
i=i.strip()
new_lst.append(i)
 return new_lst
train_df['text']=train_df['text'].apply(remove_spaces)
 test_df['text']=test_df['text'].apply(remove_spaces )
import scipy.stats as stats<load_from_csv>
df_train = pd.read_csv('.. /input/train.csv')
 df_train.shape<load_from_csv>
lemmatizer=nltk.stem.WordNetLemmatizer()
def lemmatzation(lst):
new_lst=[]
for i in lst:
i=lemmatizer.lemmatize(i)
 new_lst.append(i)
 return new_lst
train_df['text']=train_df['text'].apply(lemmatzation)
 test_df['text']=test_df['text'].apply(lemmatzation )
df_test = pd.read_csv('.. /input/test.csv')
 df_test.shape<concatenate>
train_df['text']=train_df['text'].apply(lambda x: ''.join(i+' ' for i in x))
test_df['text']=test_df['text'].apply(lambda x: ''.join(i+' ' for i in x))
df = pd.concat(( df_train, df_test), ignore_index= True, sort = False)
 df.shape<count_unique_values>
vec=CountVectorizer(ngram_range=(1,2))
train_1=vec.fit_transform(train_df['text'])
 test_1=vec.transform(test_df['text'] )
unique_hh_heads =(df_train['parentesco1'] == 1 ).sum()
unique_hh = len(df_train['idhogar'].unique())
 print('There are {} unique households and the dataset contain {} records of household heads'.format(unique_hh, unique_hh_heads))
print('There is {} households without household head'.format(unique_hh - unique_hh_heads))<count_unique_values>
tfidf=TfidfVectorizer(ngram_range=(1,2))
train_2=tfidf.fit_transform(train_df['text'])
 test_2=tfidf.transform(test_df['text'] )
unique_hh_heads =(df_test['parentesco1'] == 1 ).sum()
unique_hh = len(df_test['idhogar'].unique())
 print('There are {} unique households and the dataset contain {} records of household heads'.format(unique_hh, unique_hh_heads))
print('There is {} households without household head'.format(unique_hh - unique_hh_heads))<filter>
def select_model(x,y):
models=[{
'name':'Multinomial Naive Bayes',
'estimator':MultinomialNB() ,
'hyperparameters':{
'alpha':np.arange(0.1,1,0.01)
 }}]
for i in models:
print(i['name'])
 gs=GridSearchCV(i['estimator'], param_grid=i['hyperparameters'], cv=10, scoring='f1')
 gs.fit(x,y)
 print(gs.best_score_)
 print(gs.best_params_)
 select_model(train_2, train_df['target'] )
df.isnull().sum() [df.isnull().sum() > 0]<count_values>
clf_NB=MultinomialNB()
clf_NB.fit(train_2,train_df['target'])
 pred=clf_NB.predict(test_2)
 pred[:15]
<define_variables><EOS> 
submission_df = {"id":test_df['id'],
"target":pred}
submission = pd.DataFrame(submission_df)
 submission.to_csv('submission_df.csv',index=False )