Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<count_missing_values>
nltk.download('stopwords')
 nltk.download('punkt')
 nltk.download('averaged_perceptron_tagger')
 nltk.download('wordnet')
 !pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.2.5/en_core_web_sm-2.2.5.tar.gz
stop = set(stopwords.words('english'))
plt.style.use('fivethirtyeight')
 sns.set(font_scale=1.5)
 pd.options.display.max_columns = 250
pd.options.display.max_rows = 250
warnings.filterwarnings('ignore')
 seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val )
df['v2a1'].isnull().sum()<feature_engineering>
trainv = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 testv = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' )
df['temp_tipovivi'] = df[col].idxmax(axis = 1 )<count_values>
def remove_URL(text):
url = re.compile(r'https?://\S+|www\.\S+')
 return url.sub(r'', text)
 def remove_emoji(text):
emoji_pattern = re.compile(
 '['
u'\U0001F600-\U0001F64F'
u'\U0001F300-\U0001F5FF'
u'\U0001F680-\U0001F6FF'
u'\U0001F1E0-\U0001F1FF'
u'\U00002702-\U000027B0'
u'\U000024C2-\U0001F251'
']+',
flags=re.UNICODE)
 return emoji_pattern.sub(r'', text)
 def remove_html(text):
html = re.compile(r'<.*?>|&([a-z0-9]+|
return re.sub(html, '', text)
 def remove_punct(text):
table = str.maketrans('', '', string.punctuation)
 return text.translate(table)
 trainv['text_clean'] = trainv['text'].apply(lambda x: remove_URL(x))
trainv['text_clean'] = trainv['text_clean'].apply(lambda x: remove_emoji(x))
trainv['text_clean'] = trainv['text_clean'].apply(lambda x: remove_html(x))
trainv['text_clean'] = trainv['text_clean'].apply(lambda x: remove_punct(x))
df['temp_tipovivi'][df['v2a1'].isnull() ].value_counts()<data_type_conversions>
trainv['tokenized'] = trainv['text_clean'].apply(word_tokenize)
 trainv.head()
df['v2a1'].fillna(value = 0, inplace = True )<count_values>
trainv['lower'] = trainv['tokenized'].apply(
 lambda x: [word.lower() for word in x])
 trainv.head()
df['temp_tipovivi'][df['v2a1'] == 0].value_counts()<define_variables>
trainv['stopwords_removed'] = trainv['lower'].apply(
 lambda x: [word for word in x if word not in stop])
 trainv.head()
tipovivi2 =(df['v2a1'] == 0)&(df['tipovivi2'] == 1)
 tipovivi3 =(df['v2a1'] == 0)&(df['tipovivi3'] == 1 )<feature_engineering>
trainv['pos_tags'] = trainv['stopwords_removed'].apply(nltk.tag.pos_tag)
 trainv.head()
df.loc[tipovivi2,'tipovivi1'] = 1
df.loc[tipovivi3,'tipovivi1'] = 1
df.loc[tipovivi2,'tipovivi2'] = 0
df.loc[tipovivi3,'tipovivi3'] = 0<feature_engineering>
def get_wordnet_pos(tag):
if tag.startswith('J'):
return wordnet.ADJ
elif tag.startswith('V'):
return wordnet.VERB
elif tag.startswith('N'):
return wordnet.NOUN
elif tag.startswith('R'):
return wordnet.ADV
else:
return wordnet.NOUN
trainv['wordnet_pos'] = trainv['pos_tags'].apply(
 lambda x: [(word, get_wordnet_pos(pos_tag)) for(word, pos_tag)in x])
 trainv.head()
df['temp_tipovivi'] = df[col].idxmax(axis = 1 )<create_dataframe>
wnl = WordNetLemmatizer()
trainv['lemmatized'] = trainv['wordnet_pos'].apply(
 lambda x: [wnl.lemmatize(word, tag)for word, tag in x])
 trainv['lemmatized'] = trainv['lemmatized'].apply(
 lambda x: [word for word in x if word not in stop])
 trainv['lemma_str'] = [' '.join(map(str, l)) for l in trainv['lemmatized']]
trainv.head()
df.pivot_table(values = 'idhogar' , index = 'Target', columns = 'temp_tipovivi', aggfunc= 'count', margins= True )<count_missing_values>
lis = [
trainv[trainv['target'] == 0]['lemma_str'],
trainv[trainv['target'] == 1]['lemma_str']
]
df['v18q'][df['v18q1'].isnull() ].value_counts()<data_type_conversions>
fig, axes = plt.subplots(1, 2, figsize=(18, 8))
axes = axes.flatten()
for i, j in zip(lis, axes):
try:
new = i.str.split()
new = new.values.tolist()
corpus = [word.lower() for i in new for word in i]
dic = defaultdict(int)
 for word in corpus:
if word in stop:
dic[word] += 1
top = sorted(dic.items() , key=lambda x: x[1], reverse=True)[:15]
x, y = zip(*top)
 df = pd.DataFrame([x, y] ).T
df = df.rename(columns={0: 'Stopword', 1: 'Count'})
 sns.barplot(x='Count', y='Stopword', data=df, palette='plasma', ax=j)
 plt.tight_layout()
except:
plt.close()
print('No stopwords left in texts.')
 break
df['v18q1'].fillna(value = 0, inplace = True )<groupby>
def ngrams(n, title):

 fig, axes = plt.subplots(1, 2, figsize=(18, 8))
axes = axes.flatten()
for i, j in zip(lis, axes):
new = i.str.split()
new = new.values.tolist()
corpus = [word for i in new for word in i]
def _get_top_ngram(corpus, n=None):
vec = CountVectorizer(ngram_range=(n, n),
max_df=0.9,
stop_words='english' ).fit(corpus)
 bag_of_words = vec.transform(corpus)
 sum_words = bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0, idx])
 for word, idx in vec.vocabulary_.items() ]
words_freq = sorted(words_freq, key=lambda x: x[1], reverse=True)
 return words_freq[:15]
top_n_bigrams = _get_top_ngram(i, n)[:15]
x, y = map(list, zip(*top_n_bigrams))
sns.barplot(x=y, y=x, palette='plasma', ax=j)
 axes[0].set_title('Non Disaster Tweets')
 axes[1].set_title('Disaster Tweets')
 axes[0].set_xlabel('Count')
 axes[0].set_ylabel('Words')
 axes[1].set_xlabel('Count')
 axes[1].set_ylabel('Words')
 fig.suptitle(title, fontsize=24, va='baseline')
 plt.tight_layout()
df[df['parentesco1'] == 1].groupby('Target')['v18q'].mean()<count_values>
def display_topics(text, no_top_words, topic):

 no_top_words = no_top_words
tfidf_vectorizer = TfidfVectorizer(
 max_df=0.90, min_df=25, max_features=5000, use_idf=True)
 tfidf = tfidf_vectorizer.fit_transform(text)
 tfidf_feature_names = tfidf_vectorizer.get_feature_names()
doc_term_matrix_tfidf = pd.DataFrame(
 tfidf.toarray() , columns=list(tfidf_feature_names))
nmf = NMF(n_components=10, random_state=0,
alpha=.1, init='nndsvd' ).fit(tfidf)
 print(topic)
 for topic_idx, topic in enumerate(nmf.components_):
print('Topic %d:' %(topic_idx+1))
print(' '.join([tfidf_feature_names[i]
for i in topic.argsort() [:-no_top_words - 1:-1]]))
display_topics(lis[0], 10, 'Non Disaster Topics
' )
df['rez_esc'].value_counts(dropna = False )<count_values>
mask = np.array(Image.open('/kaggle/input/twittermaskn/twittermask.png'))
mask[mask.sum(axis=2)== 0] = 255
df['age'][df['rez_esc'].notna() ].value_counts().sort_index()<data_type_conversions>
nlp = en_core_web_sm.load()
df['rez_esc'].fillna(value = 0, inplace = True )<feature_engineering>
if torch.cuda.is_available() :
device = torch.device('cuda')
 print('There are %d GPU(s)available.' % torch.cuda.device_count())
 print('We will use the GPU:', torch.cuda.get_device_name(0))
else:
print('No GPU available, using the CPU instead.')
 device = torch.device('cpu' )
df.loc[df['rez_esc'] > 50, 'rez_esc'] = 0<count_unique_values>
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 print(f'Number of training tweets: {train.shape[0]}
')
 print(f'Number of training tweets: {test.shape[0]}
')
 display(train.sample(10))
na_mean_households = df['idhogar'][df['meaneduc'].isna() ].unique()<groupby>
labels = train['target'].values
idx = len(labels)
 combined = pd.concat([train, test])
 combined = combined.text.values
df[df['meaneduc'].isna() ].groupby('idhogar')['age'].max()<groupby>
tokenizer = BertTokenizer.from_pretrained('bert-large-uncased', do_lower_case=True )
mapper = df[df['age'] >= 18].groupby('idhogar')['escolari'].mean().to_dict()<feature_engineering>
print(' Original: ', combined[0])
 print('Tokenized: ', tokenizer.tokenize(combined[0]))
print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(combined[0])) )
df['meaneduc'] = df.apply(lambda x: mapper.get('idhogar', 0)if np.isnan(x['meaneduc'])else x['meaneduc'], axis = 1 )<count_missing_values>
max_len = 0
for text in combined:
input_ids = tokenizer.encode(text, add_special_tokens=True)
 max_len = max(max_len, len(input_ids))
print('Max sentence length: ', max_len )
df['meaneduc'].isna().sum()<feature_engineering>
token_lens = []
for text in combined:
tokens = tokenizer.encode(text, max_length = 512)
 token_lens.append(len(tokens))
df['SQBmeaned'] = df['meaneduc']**2<groupby>
train= combined[:idx]
test = combined[idx:]
train.shape
df[df['parentesco1'] == 1].groupby('Target')['hacdor'].mean()<drop_column>
def tokenize_map(sentence,labs='None'):

 global labels
input_ids = []
attention_masks = []
for text in sentence:
encoded_dict = tokenizer.encode_plus(
 text,
add_special_tokens = True,
truncation='longest_first',
max_length = 84,
pad_to_max_length = True,
return_attention_mask = True,
return_tensors = 'pt', 
)
 input_ids.append(encoded_dict['input_ids'])
 attention_masks.append(encoded_dict['attention_mask'])
 input_ids = torch.cat(input_ids, dim=0)
 attention_masks = torch.cat(attention_masks, dim=0)
 if labs != 'None':
labels = torch.tensor(labels)
 return input_ids, attention_masks, labels
else:
return input_ids, attention_masks
df.drop(columns= ['tamhog', 'hogar_total', 'r4t3'], inplace = True )<filter>
input_ids, attention_masks, labels = tokenize_map(train, labels)
 test_input_ids, test_attention_masks= tokenize_map(test )
( df['v14a'][df['parentesco1']== 1] ).mean()<groupby>
dataset = TensorDataset(input_ids, attention_masks, labels)
 train_size = int(0.8 * len(dataset))
val_size = len(dataset)- train_size
train_dataset, val_dataset = random_split(dataset, [train_size, val_size])
 print('{:>5,} training samples'.format(train_size))
print('{:>5,} validation samples'.format(val_size))
df[df['parentesco1'] == 1].groupby('Target')['v14a'].mean()<filter>
batch_size = 32
train_dataloader = DataLoader(
 train_dataset,
sampler = RandomSampler(train_dataset),
batch_size = batch_size 
)
 validation_dataloader = DataLoader(
 val_dataset,
sampler = SequentialSampler(val_dataset),
batch_size = batch_size
)
( df['refrig'][df['parentesco1']== 1] ).mean()<groupby>
prediction_data = TensorDataset(test_input_ids, test_attention_masks)
 prediction_sampler = SequentialSampler(prediction_data)
 prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size )
df[df['parentesco1'] == 1].groupby('Target')['refrig'].mean()<feature_engineering>
model = BertForSequenceClassification.from_pretrained(
 'bert-large-uncased',
num_labels = 2,
output_attentions = False,
output_hidden_states = False, 
)
 model.to(device )
df['temp_pared'] = df[col].idxmax(axis = 1 )<feature_engineering>
params = list(model.named_parameters())
 print('The BERT model has {:} different named parameters.
 '.format(len(params)))
 print('==== Embedding Layer ====
')
 for p in params[0:5]:
print('{:<55} {:>12}'.format(p[0], str(tuple(p[1].size()))))
print('
==== First Transformer ====
')
 for p in params[5:21]:
print('{:<55} {:>12}'.format(p[0], str(tuple(p[1].size()))))
print('
==== Output Layer ====
')
 for p in params[-4:]:
print('{:<55} {:>12}'.format(p[0], str(tuple(p[1].size()))))
df['temp_piso'] = df[col].idxmax(axis = 1 )<feature_engineering>
optimizer = AdamW(model.parameters() ,
lr = 6e-6,
eps = 1e-8
)
df['temp_techo'] = df[col].idxmax(axis = 1 )<groupby>
def flat_accuracy(preds, labels):

 pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return accuracy_score(labels_flat, pred_flat)
 def flat_f1(preds, labels):

 pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return f1_score(labels_flat, pred_flat )
df[df['parentesco1'] == 1].groupby('Target')['cielorazo'].mean()<feature_engineering>
def format_time(elapsed):

 elapsed_rounded = int(round(( elapsed)))
 return str(datetime.timedelta(seconds=elapsed_rounded))
df['temp_abastagua'] = df[col].idxmax(axis = 1 )<feature_engineering>
training_stats = []
total_t0 = time.time()
for epoch_i in range(0, epochs):
print('')
 print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
print('Training...')
 t0 = time.time()
total_train_loss = 0
model.train()
for step, batch in enumerate(train_dataloader):
if step % 50 == 0 and not step == 0:
elapsed = format_time(time.time() - t0)
 print(' Batch {:>5,} of {:>5,}.Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))
b_input_ids = batch[0].to(device ).to(torch.int64)
 b_input_mask = batch[1].to(device ).to(torch.int64)
 b_labels = batch[2].to(device ).to(torch.int64)
 model.zero_grad()
loss, logits = model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optimizer.step()
scheduler.step()
avg_train_loss = total_train_loss / len(train_dataloader)
 training_time = format_time(time.time() - t0)
 print('')
 print(' Average training loss: {0:.2f}'.format(avg_train_loss))
print(' Training epcoh took: {:}'.format(training_time))
print('')
 print('Running Validation...')
 t0 = time.time()
model.eval()
total_eval_accuracy = 0
total_eval_loss = 0
total_eval_f1 = 0
nb_eval_steps = 0
for batch in validation_dataloader:
b_input_ids = batch[0].to(device)
 b_input_mask = batch[1].to(device)
 b_labels = batch[2].to(device)
 with torch.no_grad() : 
(loss, logits)= model(b_input_ids,
token_type_ids=None,
attention_mask=b_input_mask,
labels=b_labels)
 total_eval_loss += loss.item()
logits = logits.detach().cpu().numpy()
label_ids = b_labels.to('cpu' ).numpy()
total_eval_accuracy += flat_accuracy(logits, label_ids)
 total_eval_f1 += flat_f1(logits, label_ids)
 avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)
 print(' Accuracy: {0:.2f}'.format(avg_val_accuracy))
avg_val_f1 = total_eval_f1 / len(validation_dataloader)
 print(' F1: {0:.2f}'.format(avg_val_f1))
avg_val_loss = total_eval_loss / len(validation_dataloader)
 validation_time = format_time(time.time() - t0)
 print(' Validation Loss: {0:.2f}'.format(avg_val_loss))
print(' Validation took: {:}'.format(validation_time))
training_stats.append(
 {
'epoch': epoch_i + 1,
'Training Loss': avg_train_loss,
'Valid.Loss': avg_val_loss,
'Valid.Accur.': avg_val_accuracy,
'Val_F1' : avg_val_f1,
'Training Time': training_time,
'Validation Time': validation_time
} 
)
 print('')
 print('Training complete!')
 print('Total training took {:}(h:mm:ss)'.format(format_time(time.time() -total_t0)) )
df['temp_electricity'] = df[['public', 'planpri', 'noelec', 'coopele']].idxmax(axis = 1 )<feature_engineering>
pd.set_option('precision', 2)
 df_stats = pd.DataFrame(data=training_stats)
 df_stats = df_stats.set_index('epoch')
 display(df_stats )
df['temp_sanitario'] = df[col].idxmax(axis = 1 )<feature_engineering>
print('Predicting labels for {:,} test sentences...'.format(len(test_input_ids)))
 model.eval()
predictions = []
for batch in prediction_dataloader:
batch = tuple(t.to(device)for t in batch)
 b_input_ids, b_input_mask, = batch
with torch.no_grad() :
outputs = model(b_input_ids, token_type_ids=None,
attention_mask=b_input_mask)
 logits = outputs[0]
logits = logits.detach().cpu().numpy()
predictions.append(logits)
 print(' DONE.' )
df['temp_energcocinar'] = df[col].idxmax(axis = 1 )<feature_engineering>
flat_predictions = [item for sublist in predictions for item in sublist]
flat_predictions = np.argmax(flat_predictions, axis=1 ).flatten()
df['cooking_lowEng'] =(( df['energcocinar1'] == 1)|(df['energcocinar4'] == 1)) *1<feature_engineering>
submission = pd.read_csv('/kaggle/input/nlp-getting-started/sample_submission.csv')
 submission['target'] = flat_predictions
submission.head(10 )
<feature_engineering><EOS> 
submission.to_csv('submission.csv', index=False, header=True )