Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<count_values>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
train_df['edjefe'].value_counts() 
<count_values>
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv' ).fillna(' ')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' ).fillna(' ')
 keep_train_index=train['text'].drop_duplicates().index
train=train.iloc[keep_train_index,:]
train_text = train["text"]
test_text=test['text']
all_text = pd.concat([train_text, test_text] )
train_df['edjefa'].value_counts() 
<count_values>
def replace_contractions(tweet):
tweet = re.sub(r"he's", "he is", tweet)
 tweet = re.sub(r"there's", "there is", tweet)
 tweet = re.sub(r"We're", "We are", tweet)
 tweet = re.sub(r"That's", "That is", tweet)
 tweet = re.sub(r"won't", "will not", tweet)
 tweet = re.sub(r"they're", "they are", tweet)
 tweet = re.sub(r"Can't", "Cannot", tweet)
 tweet = re.sub(r"wasn't", "was not", tweet)
 tweet = re.sub(r"don\x89Ûªt", "do not", tweet)
 tweet = re.sub(r"aren't", "are not", tweet)
 tweet = re.sub(r"isn't", "is not", tweet)
 tweet = re.sub(r"What's", "What is", tweet)
 tweet = re.sub(r"haven't", "have not", tweet)
 tweet = re.sub(r"hasn't", "has not", tweet)
 tweet = re.sub(r"There's", "There is", tweet)
 tweet = re.sub(r"He's", "He is", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"You're", "You are", tweet)
 tweet = re.sub(r"I'M", "I am", tweet)
 tweet = re.sub(r"shouldn't", "should not", tweet)
 tweet = re.sub(r"wouldn't", "would not", tweet)
 tweet = re.sub(r"i'm", "I am", tweet)
 tweet = re.sub(r"I\x89Ûªm", "I am", tweet)
 tweet = re.sub(r"I'm", "I am", tweet)
 tweet = re.sub(r"Isn't", "is not", tweet)
 tweet = re.sub(r"Here's", "Here is", tweet)
 tweet = re.sub(r"you've", "you have", tweet)
 tweet = re.sub(r"you\x89Ûªve", "you have", tweet)
 tweet = re.sub(r"we're", "we are", tweet)
 tweet = re.sub(r"what's", "what is", tweet)
 tweet = re.sub(r"couldn't", "could not", tweet)
 tweet = re.sub(r"we've", "we have", tweet)
 tweet = re.sub(r"it\x89Ûªs", "it is", tweet)
 tweet = re.sub(r"doesn\x89Ûªt", "does not", tweet)
 tweet = re.sub(r"It\x89Ûªs", "It is", tweet)
 tweet = re.sub(r"Here\x89Ûªs", "Here is", tweet)
 tweet = re.sub(r"who's", "who is", tweet)
 tweet = re.sub(r"I\x89Ûªve", "I have", tweet)
 tweet = re.sub(r"y'all", "you all", tweet)
 tweet = re.sub(r"can\x89Ûªt", "cannot", tweet)
 tweet = re.sub(r"would've", "would have", tweet)
 tweet = re.sub(r"it'll", "it will", tweet)
 tweet = re.sub(r"we'll", "we will", tweet)
 tweet = re.sub(r"wouldn\x89Ûªt", "would not", tweet)
 tweet = re.sub(r"We've", "We have", tweet)
 tweet = re.sub(r"he'll", "he will", tweet)
 tweet = re.sub(r"Y'all", "You all", tweet)
 tweet = re.sub(r"Weren't", "Were not", tweet)
 tweet = re.sub(r"Didn't", "Did not", tweet)
 tweet = re.sub(r"they'll", "they will", tweet)
 tweet = re.sub(r"they'd", "they would", tweet)
 tweet = re.sub(r"DON'T", "DO NOT", tweet)
 tweet = re.sub(r"That\x89Ûªs", "That is", tweet)
 tweet = re.sub(r"they've", "they have", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"should've", "should have", tweet)
 tweet = re.sub(r"You\x89Ûªre", "You are", tweet)
 tweet = re.sub(r"where's", "where is", tweet)
 tweet = re.sub(r"Don\x89Ûªt", "Do not", tweet)
 tweet = re.sub(r"we'd", "we would", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"weren't", "were not", tweet)
 tweet = re.sub(r"They're", "They are", tweet)
 tweet = re.sub(r"Can\x89Ûªt", "Cannot", tweet)
 tweet = re.sub(r"you\x89Ûªll", "you will", tweet)
 tweet = re.sub(r"I\x89Ûªd", "I would", tweet)
 tweet = re.sub(r"let's", "let us", tweet)
 tweet = re.sub(r"it's", "it is", tweet)
 tweet = re.sub(r"can't", "cannot", tweet)
 tweet = re.sub(r"don't", "do not", tweet)
 tweet = re.sub(r"you're", "you are", tweet)
 tweet = re.sub(r"i've", "I have", tweet)
 tweet = re.sub(r"that's", "that is", tweet)
 tweet = re.sub(r"i'll", "I will", tweet)
 tweet = re.sub(r"doesn't", "does not", tweet)
 tweet = re.sub(r"i'd", "I would", tweet)
 tweet = re.sub(r"didn't", "did not", tweet)
 tweet = re.sub(r"ain't", "am not", tweet)
 tweet = re.sub(r"you'll", "you will", tweet)
 tweet = re.sub(r"I've", "I have", tweet)
 tweet = re.sub(r"Don't", "do not", tweet)
 tweet = re.sub(r"I'll", "I will", tweet)
 tweet = re.sub(r"I'd", "I would", tweet)
 tweet = re.sub(r"Let's", "Let us", tweet)
 tweet = re.sub(r"you'd", "You would", tweet)
 tweet = re.sub(r"It's", "It is", tweet)
 tweet = re.sub(r"Ain't", "am not", tweet)
 tweet = re.sub(r"Haven't", "Have not", tweet)
 tweet = re.sub(r"Could've", "Could have", tweet)
 tweet = re.sub(r"youve", "you have", tweet)
 tweet = re.sub(r"donå«t", "do not", tweet)
 return tweet
print(train_df.loc[train_df['edjefe']=='no',['SQBedjefe']]['SQBedjefe'].value_counts())
 print(train_df.loc[train_df['edjefe']=='yes',['SQBedjefe']]['SQBedjefe'].value_counts() )<feature_engineering>
custom_stop_list = []
stopword_set = stopwords.words('english')+custom_stop_list+['url']
def remove_emoji(text):
emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 remove_emoji("Omg another Earthquake 😔😔")
 def preProcess(iter):
regex_ws=re.compile("\s+")
 ret=regex_ws.sub(" ",iter)
 ret=ret.replace("&amp;","&" ).replace("&lt;","<" ).replace("&gt;",">")
 regexp="(https?:\/\/(?:www\.|(?!www)|(?:xmlns\.))[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|www\.[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|https?:\/\/(?:www\.|(?!www)) [a-zA-Z0-9]+\.[^\s]{2,}|www\.[a-zA-Z0-9]+\.[^\s]{2,})"
ret=re.sub(regexp,"url",ret)
 regexp='@[A-z0-9_]+'
ret=re.sub(regexp,"@twitterhandle",ret)
 ret=remove_emoji(ret)
 ret1=re.split("[,_, \<>!\?\.:
"=*/]+",ret)
 ret2=[word for word in ret1 if word not in stopword_set]
ret2=" ".join(ret2)
 ret2=re.sub(r"(\s\d+)"," ",ret2)
 return ret2
te=preProcess("I'll make the day a fantastic 1!!")
 print(te )
train_df['edjefa'] = train_df['edjefa'].replace(( 'yes', 'no'),(1, 0))
train_df['edjefe'] = train_df['edjefe'].replace(( 'yes', 'no'),(1, 0))
test_df['edjefa'] = test_df['edjefa'].replace(( 'yes', 'no'),(1, 0))
test_df['edjefe'] = test_df['edjefe'].replace(( 'yes', 'no'),(1, 0))<data_type_conversions>
print("TFIDF")
 word_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='word',
token_pattern=r'\w{1,}',
stop_words='english',
ngram_range=(1, 1),
norm='l2',
min_df=0,
smooth_idf=False,
preprocessor=preProcess,
max_features=15000)
 word_vectorizer.fit(all_text)
 train_word_features = word_vectorizer.transform(train_text)
 test_word_features = word_vectorizer.transform(test_text)
 char_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='char',
stop_words='english',
ngram_range=(2, 6),
norm='l2',
min_df=0,
smooth_idf=False,preprocessor=preProcess,
max_features=30000)
 char_vectorizer.fit(all_text)
 train_char_features = char_vectorizer.transform(train_text)
 test_char_features = char_vectorizer.transform(test_text)
 enc = OneHotEncoder(handle_unknown='ignore')
 train_keyword_features = enc.fit_transform(train[['keyword']].to_numpy().reshape(-1,1))
test_keyword_features = enc.transform(test[['keyword']].to_numpy().reshape(-1,1))
train_features = hstack([train_char_features, train_word_features] ).tocsr()
del train_char_features,train_word_features
test_features = hstack([test_char_features, test_word_features] ).tocsr()
del test_char_features,test_word_features
print(train_features.shape)
 print(test_features.shape )
train_df['edjefa']=train_df['edjefa'].astype(float)
 train_df['edjefe']=train_df['edjefe'].astype(float)
 test_df['edjefa']=test_df['edjefa'].astype(float)
 test_df['edjefe']=test_df['edjefe'].astype(float)
<sort_values>
def f1_metric(preds,dtrain):
ytrue=dtrain.get_label()
return 'f1_score', 1-metrics.f1_score(preds.round().astype(np.int), ytrue, average='macro' )
print("Top Training Columns having missing values:")
 missing_df = train_df.isnull().sum().to_frame()
missing_df = missing_df.sort_values(0, ascending = False)
 missing_df.head() 
<groupby>
def train_model(train_features, train_target,test_features,n_estimator):
d_test = xgb.DMatrix(test_features)
 gc.collect()
print("Modeling")
 cv_scores = []
xgb_preds = []
submission = pd.DataFrame.from_dict({'id': test['id']})
 xgb_params = {'eta': 0.1,
'max_depth': 7,
'subsample': 0.7,
'colsample_bytree': 0.75,
'n_estimators':n_estimator,
'objective': 'binary:logistic',
'metric': 'f1_score',
'eval_metric':'error',
'gamma':0,
'seed': 314159265,
'verbose':20,
'min_child_weight':1
}
cv=StratifiedKFold(n_splits=4)
 for i,(train_index, valid_index)in enumerate(cv.split(train_features, train_target)) :
X_train, X_valid=train_features[train_index],train_features[valid_index]
y_train, y_valid = train_target.iloc[train_index],train_target.iloc[valid_index]
d_train = xgb.DMatrix(X_train, y_train)
 d_valid = xgb.DMatrix(X_valid, y_valid)
 watchlist = [(d_valid, 'valid')]
model = xgb.train(xgb_params, d_train, 200, watchlist, verbose_eval=False,early_stopping_rounds=100, feval=f1_metric)
 print(model.attributes())
 print(model.attributes() ['best_msg'])
 cv_scores.append(float(model.attributes() ['best_score']))
submission[i] = model.predict(d_test)
 gc.collect()
print('Total CV score is {}'.format(np.mean(cv_scores)))

print(model.attributes())
 print(model.attributes() ['best_msg'])
 cv_scores.append(float(model.attributes() ['best_score']))
submission = model.predict(d_test)
 return submission
train_df.groupby('v18q')['v18q1'].apply(lambda x: x.isnull().sum() )<data_type_conversions>
train_target = train['target']
print(train_target.shape)
 train_target.value_counts()
train_df['v18q1'] = train_df['v18q1'].fillna(0)
 test_df['v18q1'] = test_df['v18q1'].fillna(0 )<count_missing_values>
submission_tfidf=train_model(train_features, train_target,test_features,n_estimator=97)
print(train_df.loc[train_df['rez_esc'].isnull() ]['age'].value_counts().head(6))
print(train_df.loc[train_df['rez_esc'].isnull() ]['instlevel1'].value_counts())
 print(train_df.loc[train_df['rez_esc'].isnull() ]['instlevel2'].value_counts() )<feature_engineering>
embed = hub.load("https://tfhub.dev/google/universal-sentence-encoder/4")
 train_text=train_text.apply(remove_emoji)
 test_text=test_text.apply(remove_emoji)
 X_train_embeddings = embed(train_text)
 X_test_embeddings = embed(test_text)
 X_feat=sparse.csr_matrix(X_train_embeddings.numpy())
 X_test=sparse.csr_matrix(X_test_embeddings.numpy())
 X_feat.shape
submission_hub=train_model(X_feat,train_target,X_test,n_estimator=100)
<count_missing_values><EOS> 
submission=(0.35*submission_tfidf+0.65*submission_hub ).round().astype(np.int)
 final_submit = pd.DataFrame.from_dict({'id': test['id']})
 final_submit['target']=submission
print(final_submit)
 final_submit.to_csv('submission.csv', index=False)
 print(final_submit['target'].value_counts() )