Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<groupby>
%matplotlib inline
tqdm.pandas()
train_df.groupby('Target' ).mean()<feature_engineering>
train=pd.read_csv(".. /input/nlp-getting-started/train.csv")
 test=pd.read_csv(".. /input/nlp-getting-started/test.csv")
 sample=pd.read_csv(".. /input/nlp-getting-started/sample_submission.csv" )
train_df['v18q+etecho3'] = train_df['v18q']+train_df['etecho3']
print(train_df.corr() ['Target']['v18q'])
 print(train_df.corr() ['Target']['etecho3'])
 print(train_df.corr() ['Target']['v18q+etecho3'])
 test_df['v18q+etecho3'] = test_df['v18q']+test_df['etecho3']<feature_engineering>
def load_embedding(file):
def get_coefs(word,*arr):
return word,np.array(arr,dtype='float32')
 embeddings_index=dict(get_coefs(*o.split(" ")) for o in open(file)if len(o)>100)
 return embeddings_index
train_df['v18q+paredblolad'] = train_df['v18q']+train_df['paredblolad']
print(train_df.corr() ['Target']['v18q'])
 print(train_df.corr() ['Target']['paredblolad'])
 print(train_df.corr() ['Target']['v18q+paredblolad'])
 test_df['v18q+paredblolad'] = test_df['v18q']+test_df['paredblolad']<feature_engineering>
glove='.. /input/glovetwitter27b100dtxt/glove.twitter.27B.200d.txt'
train_df['v18q+pisomoscer'] = train_df['v18q']+train_df['pisomoscer']
print(train_df.corr() ['Target']['v18q'])
 print(train_df.corr() ['Target']['pisomoscer'])
 print(train_df.corr() ['Target']['v18q+pisomoscer'])
 test_df['v18q+pisomoscer'] = test_df['v18q']+test_df['pisomoscer']<feature_engineering>
embed_glove = load_embedding(glove )
train_df['pisomoscer+instlevel8'] = train_df['pisomoscer']+train_df['instlevel8']
print(train_df.corr() ['Target']['pisomoscer'])
 print(train_df.corr() ['Target']['instlevel8'])
 print(train_df.corr() ['Target']['pisomoscer+instlevel8'])
 test_df['pisomoscer+instlevel8'] = test_df['pisomoscer']+test_df['instlevel8']<feature_engineering>
def build_vocab(text):
vocab={}
for s in tqdm(text):
for word in s:
try:
vocab[word]+=1
except KeyError:
vocab[word]=1
return vocab
train_df['edjefe+escolari'] = train_df['edjefe']+train_df['escolari']
print(train_df.corr() ['Target']['edjefe'])
 print(train_df.corr() ['Target']['escolari'])
 print(train_df.corr() ['Target']['edjefe+escolari'])
 test_df['edjefe+escolari'] = test_df['edjefe']+test_df['escolari']<set_options>
def check_coverage(vocab,embedding_index):
known_words={}
unknown_words={}
nb_known_words=0
nb_unknown_words=0
for word in tqdm(vocab.keys()):
try:
known_words[word]=embedding_index[word]
nb_known_words+=vocab[word]
except:
unknown_words[word]=vocab[word]
nb_unknown_words+=vocab[word]
print('Found embeddings for {:.3%} of vocab'.format(len(known_words)/ len(vocab)))
 print('Found embeddings for {:.3%} of all text'.format(nb_known_words /(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words
pip install plotly chart-studio
cufflinks.go_offline(connected=True)
 init_notebook_mode(connected=True )<prepare_x_and_y>
sentences = train["text"].progress_apply(lambda x: x.split() ).values
vocab = build_vocab(sentences)
 print({k: vocab[k] for k in list(vocab)[:5]})
train_df['Target1'] = train_df.Target
train_df.Target1 = train_df.Target1.apply(str)
 train_df.iplot(
 x='hogar_nin',
y='meaneduc',
categories="Target1",
xTitle = d.get('hogar_nin'),
yTitle = d.get('meaneduc'),
title="Number of children vs.Average education by Poverty level")
 train_df = train_df.drop(['Target1'], axis =1 )<remove_duplicates>
oov=check_coverage(vocab,embed_glove )
c1 = corrs.abs().unstack().drop_duplicates()
c1.sort_values(ascending = True )<feature_engineering>
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
train_df["Poor_materials"]=train_df['pareddes']+train_df['paredfibras']+train_df['pisonatur']+train_df['pisonotiene']+train_df['techocane']+train_df['epared1']+train_df['etecho1']+train_df['eviv1']
test_df["Poor_materials"]=test_df['pareddes']+test_df['paredfibras']+test_df['pisonatur']+test_df['pisonotiene']+test_df['techocane']+test_df['epared1']+test_df['etecho1']+test_df['eviv1']
print('Pearson correlation coefficients:')
 print('Poor Materials(training set): ',train_df['Poor_materials'].corr(train_df['Target']))<feature_engineering>
def known_contractions(embed):
known = []
for contract in contraction_mapping:
if contract in embed:
known.append(contract)
 return known
train_df["Rich_Materials"]=train_df['paredblolad']+train_df['pisomoscer']+train_df['techoentrepiso']+train_df['techootro']+train_df['cielorazo']+train_df['epared3']+train_df['etecho3']+train_df['eviv3']
test_df["Rich_Materials"]=test_df['paredblolad']+test_df['pisomoscer']+test_df['techoentrepiso']+test_df['techootro']+test_df['cielorazo']+test_df['epared3']+test_df['etecho3']+test_df['eviv3']
print('Pearson correlation coefficients:')
 print('Materials(training set): ',train_df['Rich_Materials'].corr(train_df['Target']))<feature_engineering>
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
train_df["Poor_Infrastructure"]=train_df['abastaguano']+train_df['noelec']+train_df['epared1']+train_df['etecho1']+train_df['eviv1']+train_df['lugar3']+train_df['sanitario1']+train_df['energcocinar1']+train_df['elimbasu3']
test_df["Poor_Infrastructure"]=test_df['abastaguano']+test_df['noelec']+test_df['epared1']+test_df['etecho1']+test_df['eviv1']+test_df['lugar3']+test_df['sanitario1']+test_df['energcocinar1']+test_df['elimbasu3']
print('Pearson correlation coefficients:')
 print('Materials(training set): ',train_df['Poor_Infrastructure'].corr(train_df['Target']))<feature_engineering>
train['text_clean']=train['text'].progress_apply(lambda x:clean_contractions(x,contraction_mapping))
sentence=train['text_clean'].apply(lambda x:x.split())
 vocab=build_vocab(sentence )
train_df["Good_Infrastructure"]=train_df['sanitario2']+train_df['energcocinar2']+train_df['elimbasu1']+train_df['abastaguadentro']+train_df['planpri']+train_df['epared3']+train_df['etecho3']*(3)+train_df['eviv3']+train_df['lugar1']+train_df['lugar2']+train_df['lugar6']
test_df["Good_Infrastructure"]=test_df['sanitario2']+test_df['energcocinar2']+test_df['elimbasu1']+test_df['abastaguadentro']+test_df['planpri']+test_df['epared3']+test_df['etecho3']*(3)+test_df['eviv3']+test_df['lugar1']+test_df['lugar2']+test_df['lugar6']
print('Pearson correlation coefficients:')
 print('Infrastructure(training set): ',train_df['Good_Infrastructure'].corr(train_df['Target']))<feature_engineering>
oov=check_coverage(vocab,embed_glove )
train_df["overcrowding_total"] = train_df["hacdor"]+train_df["r4t1"] +train_df["instlevel1"] + train_df["area2"]
test_df["overcrowding_total"] = test_df["hacdor"]+ test_df["r4t1"] + test_df["instlevel1"] + test_df["area2"]
print('overcrowding_total: ',train_df['overcrowding_total'].corr(train_df['Target']))<feature_engineering>
def clean_text(x):
x = str(x)
 for punct in "/-'":
x = x.replace(punct, ' ')
 for punct in '&':
x = x.replace(punct, f' {punct} ')
 for punct in '?!.,"
x = x.replace(punct, '')
 return x
train_df["escolari+hacapo"] = train_df["escolari"]+train_df["hacapo"]
test_df["escolari+hacapo"] = test_df["escolari"]+test_df["hacapo"]
print(train_df['escolari+hacapo'].corr(train_df['Target']))<install_modules>
train['text_clean']=train['text_clean'].progress_apply(lambda x:clean_text(x))
sentence=train['text_clean'].apply(lambda x:x.split())
 vocab=build_vocab(sentence )
pip install --upgrade https://github.com/featuretools/featuretools/zipball/master<define_variables>
oov=check_coverage(vocab,embed_glove )
ind_bool = list()
ind_ordered = list()
ind_cont = list()
hh_bool = list()
hh_ordered = list()
hh_cont = list()
print(train_df.drop(['age', 'SQBescolari','SQBage', 'agesq'], axis = 1 ).columns.get_loc('Target'))<data_type_conversions>
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
x = re.sub('[0-9]','
return x
test_df['Target'] = np.nan
data = train_df.append(test_df, sort = True)
 for variable in(hh_bool + ind_bool):
data[variable] = data[variable].astype('bool')
 for variable in(hh_cont + ind_cont):
data[variable] = data[variable].astype(float)
 for variable in(hh_ordered + ind_ordered):
try:
data[variable] = data[variable].astype(int)
 except Exception as e:
print(f'Could not convert {variable} because of missing values.' )<create_dataframe>
train['text_clean']=train['text_clean'].progress_apply(lambda x: clean_numbers(x))
train['text_clean']=train['text_clean'].apply(lambda x:x.lower() )
es = ft.EntitySet(id = 'households')
 es.entity_from_dataframe(entity_id = 'data',
dataframe = data,
index = 'Id')

es.normalize_entity(base_entity_id='data',
new_entity_id='household',
index = 'idhogar',
additional_variables = hh_bool + hh_ordered + hh_cont+["Target"] )<create_dataframe>
sentence=train['text_clean'].apply(lambda x:x.split())
 vocab=build_vocab(sentence)
 oov=check_coverage(vocab,embed_glove )
feature_matrix, feature_names = ft.dfs(entityset=es,
target_entity = 'household',
max_depth = 2,
verbose = 1,
n_jobs = -1,
chunk_size = 100 )<drop_column>
'disease' in embed_glove
drop_cols = []
for col in feature_matrix:
if col == 'Target':
pass
else:
if 'Target' in col:
drop_cols.append(col)
 print(drop_cols)
 feature_matrix = feature_matrix[[x for x in feature_matrix if x not in drop_cols]]
feature_matrix.head()<split>
mispell_dict={'mh
'
're
'
'diseasese':'disease',
'don
'
'
'mediterraneanean':'mediterranean',
'it
'
'you
'chinaªs':'china',
'be
'ices
's
'by
'iªm':'I am',
'
'read
'enchanged':'exchanged',
'let
'can
'americaªs':'america',
'of
'here
'grief
'
'idfire':'id fire',
'karymsky':'sky',
'rexyy':'rex',
'japìn':'japan',
'
'carryi':'carrying',
'offensive
'
'lulgzimbestpicts':'pics',
'ramag':'ram',
'diamondkesawn':'diamond',
'raynbowaffair':'rainbow affair',
'viwxy
'lvlh
'yazidis':'yazid',
'
'waimate':'mate',
'otrametlife':'
'
'
'
'rqwuoy
'ks
'
'iªve':'I have',
'unsuckdcmetro':'metro',
'realdonaldtrump':'donald trump',
'abbswinston':'winston',
'moll
'oppressions':'oppression',
'slanglucci':'slang',
'fettilootch':'
'worstsummerjob':'worst summer job',
'
'i
'åê':'
'åè':'
'
'warfighting':'war fighting',
'mitt
'
'michael
'kurtschlichter':'
'beforeitsnews':'before it is news',
'zujwuiomb':'
'lonewolffur':'lone wolf',
'm
'localarsonist':'local arson',
'wvj
'thoyhrhkfj':'
'o
'nnmqlzo
'temporary
'pbban':'
'm
'votejkt
'rockyfire':'rock fire',
'throwingknifes':'throwing knife',
'dannyonpc':'danny on picture',
'godslove':'god love',
'ptsdchat':'chat',
'cbcca':'
'usnwsgov':'us government',
'metrofmtalk':'metro fm talk',
'
'bookboost':'book boost',
'ibooklove':'book love',
'aoms':'
'foxysiren':'fox',
'blowmandyup':'blow',
'dehmym
'itunesmusic':'i tunes music',
'\x89û÷politics':'politics','viralspell':'viral spell'}
train_df = feature_matrix[feature_matrix['Target'].notnull() ].reset_index()
test_df = feature_matrix[feature_matrix['Target'].isnull() ].reset_index()
test_df.head(5 )<drop_column>
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
idhogar = test_df['idhogar']
train_df = train_df.select_dtypes(exclude=['object'])
 test_df = test_df.select_dtypes(exclude=['object'])
 train_df = train_df.dropna(axis='columns')
 test_df = test_df.dropna(axis='columns' )<prepare_x_and_y>
train['text_clean']=train['text_clean'].progress_apply(lambda x: correct_spelling(x,mispell_dict))
sentence=train['text_clean'].apply(lambda x:x.split())
 vocab=build_vocab(sentence)
 oov=check_coverage(vocab,embed_glove)
corr_matrix = train_df.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(abs(upper[column])> 0.99)]
print(f'There are {len(to_drop)} correlated columns to remove.')
 print(to_drop)
 train_df = train_df.drop(columns = to_drop )<prepare_x_and_y>
test['text_clean']=test['text'].progress_apply(lambda x:clean_contractions(x,contraction_mapping))
test['text_clean']=test['text_clean'].progress_apply(lambda x:clean_text(x))
test['text_clean']=test['text_clean'].progress_apply(lambda x: clean_numbers(x))
test['text_clean']=test['text_clean'].apply(lambda x:x.lower())
 test['text_clean']=test['text_clean'].progress_apply(lambda x: correct_spelling(x,mispell_dict))
sentence=test['text_clean'].apply(lambda x:x.split())
 vocab=build_vocab(sentence)
 oov=check_coverage(vocab,embed_glove)
train_df_H20 = train_df
y_df = train_df['Target']
train_df, test_df = train_df.align(test_df, join = 'inner', axis = 1)
 print(f"Training set shape:{train_df.shape}, testing set shape:{test_df.shape}" )<prepare_x_and_y>
train_df,val_df=train_test_split(train,test_size=0.1,random_state=100 )
X = train_df.values
y = y_df.values
y = y.reshape(-1, 1)
 test_np = test_df.values
X.shape<split>
embed_size=300
max_features=20000
maxlen=300
X_train, X_test, y_train, y_test = train_test_split(X, y,test_size = 0.1, random_state = 123)
 X_train.shape<normalization>
train_X=train_df['text_clean'].values
valid_X=val_df['text_clean'].values
test_X=test['text_clean'].values
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
 X_test = sc.transform(X_test)
 X = sc.transform(X)
 print(X)
 test_np = sc.transform(test_np)
 print(test_np )<train_model>
tokenizer=Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X=tokenizer.texts_to_sequences(train_X)
 valid_X=tokenizer.texts_to_sequences(valid_X)
 test_X=tokenizer.texts_to_sequences(test_X)
 train_X=pad_sequences(train_X,maxlen=maxlen)
 valid_X=pad_sequences(valid_X,maxlen=maxlen)
 test_X=pad_sequences(test_X,maxlen=maxlen)
 train_Y=train_df['target'].values
valid_Y=val_df['target'].values
<choose_model_class>
all_embs = np.stack(embed_glove.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
classifier = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=None, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced',
colsample_bytree = 0.93, min_child_samples = 95, num_leaves = 14, subsample = 0.96 )<train_model>
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in tqdm(word_index.items()):
if i >= max_features: continue
embedding_vector = embed_glove.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
eval_set = [(X_train, y_train),(X_test, y_test)]
classifier.fit(X_train, y_train, eval_metric="multiclass", eval_set=eval_set, verbose=True, early_stopping_rounds=400)
 y_pred = classifier.predict(X_test)
 y_pred = y_pred.reshape(-1, 1 )<compute_test_metric>
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix],trainable=False )(inp)
 x=SpatialDropout1D(0.2 )(x)
 x = Bidirectional(LSTM(128,return_sequences=True))(x)
 x=GlobalMaxPool1D()(x)
 x = Dense(16, activation="relu" )(x)
 x = Dropout(0.1 )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )
cm1 = confusion_matrix(y_test, y_pred)
 print(cm1 )<compute_test_metric>
model.fit(train_X, train_Y, batch_size=64, epochs=5, validation_data=(valid_X, valid_Y))
f1 = f1_score(y_test, y_pred, average ='macro')
 print('f1 score for LGBoost model:',f1 )<predict_on_test>
pred_glove_val_y = model.predict([valid_X], batch_size=64, verbose=1)
 print(f'F1 Score {f1_score(valid_Y,(pred_glove_val_y>0.6 ).astype(int)) }' )
y_pred = classifier.predict(test_np)
 y_pred = y_pred.reshape(-1, 1)
 y_pred = y_pred.astype(int)
 print(plt.hist(y_pred))<categorify>
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
test_df['Target'] = y_pred.astype(int)
 test_df['idhogar'] = idhogar
submit = submit.merge(test_df[['idhogar', 'Target']], on = 'idhogar', how = 'left' ).drop(columns = ['idhogar'])
<data_type_conversions>
sample['target']=(pred_glove_test_y>0.6 ).astype(int )
submit['Target'] = submit['Target'].fillna(4)
 submit['Target'] = submit['Target'].astype(int )<save_to_csv>
sample['target'].value_counts()
submit.to_csv('LGBClassification.csv', index = False )<train_model>
sample.to_csv('sample_submission.csv',index=False )