Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<prepare_output>
import numpy as np
from tqdm import tqdm_notebook
import pandas as pd
import gc
import pdb
missing = pd.DataFrame(data.isnull().sum() ).rename(columns = {0: 'total'})
 missing['percent'] = missing['total'] / len(data)
 missing.sort_values('percent', ascending = False ).head(10 ).drop('Target' )<groupby>
!pip install transformers
heads.groupby('v18q')['v18q1'].apply(lambda x: x.isnull().sum() )<data_type_conversions>
import plotly.graph_objs as go
from matplotlib import pyplot as plt
import plotly.offline as py
import regex as re
from bs4 import BeautifulSoup
import string
from nltk.corpus import stopwords
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
from nltk.tokenize import word_tokenize
from gensim.models import word2vec
data['v18q1'] = data['v18q1'].fillna(0 )<count_values>
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
data.loc[(data['tipovivi1'] == 1), 'v2a1'] = 0
data['v2a1-missing'] = data['v2a1'].isnull()
data['v2a1-missing'].value_counts()<feature_engineering>
nltk.download('punkt' )
data.loc[(( data['age'] > 19)|(data['age'] < 7)) &(data['rez_esc'].isnull()), 'rez_esc'] = 0
data['rez_esc-missing'] = data['rez_esc'].isnull()<feature_engineering>
warnings.filterwarnings('ignore' )
data.loc[data['rez_esc'] > 5, 'rez_esc'] = 5<define_variables>
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu" )
id_ = ['Id', 'idhogar', 'Target']<define_variables>
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv" )
ind_bool = ['v18q', 'dis', 'male', 'female', 'estadocivil1', 'estadocivil2', 'estadocivil3',
'estadocivil4', 'estadocivil5', 'estadocivil6', 'estadocivil7',
'parentesco1', 'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5',
'parentesco6', 'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10',
'parentesco11', 'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3',
'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8',
'instlevel9', 'mobilephone', 'rez_esc-missing']
ind_ordered = ['rez_esc', 'escolari', 'age']<define_variables>
def get_embedding(df,col):
corpus = create_corpus(df,col)
 w2v_model = word2vec.Word2Vec([corpus], size=50, window=10, min_count=1, workers=4)
 embedding_dict={}
for word in corpus:
embedding_dict[word] = w2v_model.wv[word]
num_words=len(corpus)
 embedding_matrix=np.zeros(( num_words,50))
for i,word in tqdm_notebook(enumerate(df[col].unique())) :
if i > num_words:
continue
emb_vec=embedding_dict.get(word)
 if emb_vec is not None:
embedding_matrix[i]=emb_vec
return embedding_matrix
def create_corpus(df,col):
corpus=[]
for keyword in tqdm_notebook(df[col].unique()):
keyword=keyword.lower()
corpus.append(keyword)
 return corpus
hh_bool = ['hacdor', 'hacapo', 'v14a', 'refrig', 'paredblolad', 'paredzocalo',
'paredpreb','pisocemento', 'pareddes', 'paredmad',
'paredzinc', 'paredfibras', 'paredother', 'pisomoscer', 'pisoother',
'pisonatur', 'pisonotiene', 'pisomadera',
'techozinc', 'techoentrepiso', 'techocane', 'techootro', 'cielorazo',
'abastaguadentro', 'abastaguafuera', 'abastaguano',
'public', 'planpri', 'noelec', 'coopele', 'sanitario1',
'sanitario2', 'sanitario3', 'sanitario5', 'sanitario6',
'energcocinar1', 'energcocinar2', 'energcocinar3', 'energcocinar4',
'elimbasu1', 'elimbasu2', 'elimbasu3', 'elimbasu4',
'elimbasu5', 'elimbasu6', 'epared1', 'epared2', 'epared3',
'etecho1', 'etecho2', 'etecho3', 'eviv1', 'eviv2', 'eviv3',
'tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5',
'computer', 'television', 'lugar1', 'lugar2', 'lugar3',
'lugar4', 'lugar5', 'lugar6', 'area1', 'area2', 'v2a1-missing']
hh_ordered = [ 'rooms', 'r4h1', 'r4h2', 'r4h3', 'r4m1','r4m2','r4m3', 'r4t1', 'r4t2',
'r4t3', 'v18q1', 'tamhog','tamviv','hhsize','hogar_nin',
'hogar_adul','hogar_mayor','hogar_total', 'bedrooms', 'qmobilephone']
hh_cont = ['v2a1', 'dependency', 'edjefe', 'edjefa', 'meaneduc', 'overcrowding']<define_variables>
def combine_features(df,features,cat_cols):
df_copy=df.copy()
for col in cat_cols:
embedding = get_embedding(df_copy,col)
 vec = {val:embedding[i] for i, val in enumerate(df_copy[col].unique())}
for key,value in vec.items() :
df_copy[col] = df_copy[col].map(lambda x: value if x == key else x)
 embed_array = np.stack(df_copy[col].values,axis=0)
 features = np.concatenate(( features,embed_array),axis=1)
 return features
sqr_ = ['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe',
'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned', 'agesq']<define_variables>
train['keyword'] = train.keyword.fillna(value='None')
 train['location'] = train.location.fillna(value='None')
 test['keyword'] = test.keyword.fillna(value='None')
 test['location'] = test.location.fillna(value='None' )
x = ind_bool + ind_ordered + id_ + hh_bool + hh_ordered + hh_cont + sqr_
print('There are no repeats: ', np.all(np.array(list(Counter(x ).values())) == 1))
print('We covered every variable: ', len(x)== data.shape[1] )<filter>
def get_features(data, batch_size=2500):
model_class, tokenizer_class, pretrained_weights =(ppb.DistilBertModel, ppb.DistilBertTokenizer, 'distilbert-base-uncased')
 tokenizer = tokenizer_class.from_pretrained(pretrained_weights)
 model = model_class.from_pretrained(pretrained_weights)
 model.to(device)
 model = nn.DataParallel(model)
 tokenized = data["text"].apply(( lambda x: tokenizer.encode(x, add_special_tokens=True)))
 max_len = 0
for i in tokenized.values:
if len(i)> max_len:
max_len = len(i)
 padded = np.array([i + [0]*(max_len-len(i)) for i in tokenized.values])
 attention_mask = np.where(padded != 0, 1, 0)
 last_hidden_states=[]
no_batch = data.shape[0]//batch_size
start_index=0
end_index=1
for i in tqdm_notebook(range(1,no_batch+2)) :
if data.shape[0]>batch_size*i:
end_index=batch_size*i
else:
end_index=train.shape[0]
input_ids = torch.tensor(padded[start_index:end_index])
 batch_attention_mask = torch.tensor(attention_mask[start_index:end_index])
 input_ids = input_ids.to(device)
 batch_attention_mask = batch_attention_mask.to(device)
 with torch.no_grad() :
batch_hidden_state = model(input_ids, attention_mask=batch_attention_mask)
 print("Batch {} is completed sucessfully".format(i))
last_hidden_states.append(batch_hidden_state[0])
 start_index=batch_size*i
end_index=batch_size*i
fin_features = torch.cat(last_hidden_states,0)
 clf_features = fin_features[:,0,:].cpu().numpy()
return clf_features
corr_matrix.loc[corr_matrix['tamhog'].abs() > 0.9, corr_matrix['tamhog'].abs() > 0.9]<drop_column>
gc.collect()
features = get_features(train,batch_size=2500)
 test_distil_features = get_features(test,batch_size=2500 )
heads = heads.drop(columns = ['tamhog', 'hogar_total', 'r4t3'] )<feature_engineering>
cat_cols = ['keyword']
heads['phones-per-capita'] = heads['qmobilephone'] / heads['tamviv']
heads['tablets-per-capita'] = heads['v18q1'] / heads['tamviv']
heads['rooms-per-capita'] = heads['rooms'] / heads['tamviv']
heads['rent-per-capita'] = heads['v2a1'] / heads['tamviv']<drop_column>
train_features = combine_features(train,features,cat_cols)
 test_features = combine_features(test,test_distil_features,cat_cols )
ind = ind.drop(columns = 'male' )<groupby>
labels = train["target"]
range_ = lambda x: x.max() - x.min()
range_.__name__ = 'range_'
ind_agg = ind.drop(columns = 'Target' ).groupby('idhogar' ).agg(['min', 'max', 'sum', 'count', 'std', range_])
 ind_agg.head()<rename_columns>
lr_clf = LogisticRegression()
lr_clf.fit(train_features, labels )
new_col = []
for c in ind_agg.columns.levels[0]:
for stat in ind_agg.columns.levels[1]:
new_col.append(f'{c}-{stat}')
 ind_agg.columns = new_col
ind_agg.head()<merge>
test_pred = lr_clf.predict(test_features )
<import_modules><EOS> 
submission['target'] = test_pred
submission.to_csv('submission.csv', index=False )