Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<feature_engineering>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from pathlib import Path
import os, re, string
heads['phones-per-capita'] = heads['qmobilephone'] / heads['tamviv']
heads['tablets-per-capita'] = heads['v18q1'] / heads['tamviv']
heads['rooms-per-capita'] = heads['rooms'] / heads['tamviv']
heads['rent-per-capita'] = heads['v2a1'] / heads['tamviv']<merge>
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
corrs = pcorrs.merge(scorrs, on = 'feature')
 corrs['diff'] = corrs['pcorr'] - corrs['scorr']
corrs.sort_values('diff' ).head()<data_type_conversions>
train_df = pd.read_csv(path/'train.csv')
 test_df = pd.read_csv(path/'test.csv' )
corr_matrix = ind.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape),k=1 ).astype(np.bool))
to_drop = [col for col in upper.columns if any(abs(upper[col])> 0.95)]
to_drop<drop_column>
print(f'Training set size: {len(train_df)}')
 print(f'Test set size: {len(test_df)}' )
ind = ind.drop(columns = 'male' )<drop_column>
print(f'Missing values in training set: {train_df.text.isna().sum() }')
 print(f'Missing values in test set: {test_df.text.isna().sum() }' )
ind = ind.drop(columns = [c for c in ind if c.startswith('instlevel')])
 ind.shape<groupby>
print(f'Average target in training set: {np.round(train_df.target.mean() ,2)}' )
range_ = lambda x:x.max() - x.min()
range_.__name__ = 'range_'
ind_agg = ind.drop(columns='Target' ).groupby('idhogar' ).agg(['min','max','sum','count','std',range_])
 ind_agg.head()<feature_engineering>
diff = len(train_df[train_df.target==0])-len(train_df[train_df.target==1])
 pct_diff = np.round(diff/len(train_df),2)
 diff, pct_diff
new_cols = []
for col in ind_agg.columns.levels[0]:
for stat in ind_agg.columns.levels[1]:
new_cols.append(f'{col}-{stat}')
 ind_agg.columns = new_cols
ind_agg.head()<merge>
re_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤€‘’])')
 def re_tokenizer(s): return re_tok.sub(r' \1 ', s ).lower().split()
ind_agg = ind_agg.drop(columns=to_drop)
 ind_feats = list(ind_agg.columns)
 final = heads.merge(ind_agg,on='idhogar',how='left')
 print('Final features shape: ', final.shape )<merge>
print(re_tokenizer(train_df.text[6]))
head_gender = ind.loc[ind['parentesco1'] == 1, ['idhogar', 'female']]
final = final.merge(head_gender, on = 'idhogar', how = 'left' ).rename(columns = {'female': 'female-head'} )<count_values>
X_train, X_valid, y_train, y_valid = train_test_split(train_df.text, train_df.target, test_size=0.1, random_state=42 )
final.groupby('female-head')['Target'].value_counts(normalize=True )<groupby>
print(f'Training set size: {len(X_train)}')
 print(f'Validation set size: {len(X_valid)}' )
final.groupby('female-head')['meaneduc'].agg(['mean', 'count'] )<import_modules>
vec = CountVectorizer(ngram_range=(1,2), tokenizer=re_tokenizer, min_df=4, max_df=0.8, strip_accents='unicode', lowercase=False )
scorer = make_scorer(f1_score, greater_is_better=True, average = 'macro' )<data_type_conversions>
train_term_doc = vec.fit_transform(X_train)
 valid_term_doc = vec.transform(X_valid)
 train_term_doc.shape, valid_term_doc.shape
train_labels = np.array(list(final.loc[final['Target'].notnull() ,'Target'].astype(np.uint8)))
 train_set = final[final['Target'].notnull() ].drop(columns=['Id','idhogar','Target'])
 test_set = final[final['Target'].isnull() ].drop(columns=['Id','idhogar','Target'])
 submission_base = test[['Id', 'idhogar']].copy()<categorify>
x_train = train_term_doc
y_train = y_train.values
x_valid = valid_term_doc
y_valid = y_valid.values
features = list(train_set.columns)
 pipeline = Pipeline([('imputer',Imputer(strategy='median')) ,('scaler',MinMaxScaler())])
 train_set = pipeline.fit_transform(train_set)
 test_set = pipeline.transform(test_set )<compute_train_metric>
p1 = np.squeeze(np.asarray(x_train[y_train==1].sum(0)))
 p0 = np.squeeze(np.asarray(x_train[y_train==0].sum(0)) )
model = RandomForestClassifier(n_estimators=100, random_state=10, n_jobs = -1)
 cv_score = cross_val_score(model, train_set, train_labels, cv = 10, scoring = scorer)
 print(f'10 Fold Cross Validation F1 Score = {round(cv_score.mean() , 4)} with std = {round(cv_score.std() , 4)}' )<train_model>
pr1 =(p1+1)/(( y_train==1 ).sum() +1)
 pr0 =(p0+1)/(( y_train==0 ).sum() +1 )
model.fit(train_set,train_labels)
 indices = np.argsort(model.feature_importances_)[::-1]
importances = model.feature_importances_
for i in range(len(features)) :
print("%d)feature importances of %s %f"%(i,features[indices[i]],importances[indices[i]]))<create_dataframe>
pr1[2164] / pr0[2164]
temp_df = pd.DataFrame({'feature':features,'importance':importances} )<import_modules>
b = np.log(( y_train==1 ).mean() /(y_train==0 ).mean()); b
from sklearn.svm import LinearSVC
from sklearn.naive_bayes import GaussianNB
from sklearn.neural_network import MLPClassifier
from sklearn.linear_model import LogisticRegressionCV, RidgeClassifierCV
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.neighbors import KNeighborsClassifier<compute_train_metric>
preds =(x_valid @ r + b)> 0
model_results = cv_model(train_set,train_labels,LinearSVC() ,'LSVC',model_results )<compute_train_metric>
print(f'Validation accuracy: {(preds == y_valid ).mean() }')
 print(f'Validation F1 score: {f1_score(y_valid, preds)}' )
model_results = cv_model(train_set,train_labels,GaussianNB() ,'GNB',model_results )<compute_train_metric>
vec_tfidf = TfidfVectorizer(ngram_range=(1,2), tokenizer=re_tokenizer, lowercase=False,
min_df=4, max_df=0.8, strip_accents='unicode', sublinear_tf=True )
model_results = cv_model(train_set,train_labels,MLPClassifier(hidden_layer_sizes=(32,64,128,64,32)) ,'MLP',model_results )<compute_test_metric>
train_term_doc_tfidf = vec_tfidf.fit_transform(X_train)
 valid_term_doc_tfidf = vec_tfidf.transform(X_valid )
model_results = cv_model(train_set, train_labels,
LinearDiscriminantAnalysis() ,
'LDA', model_results )<compute_train_metric>
def pr(y_i, y, x):
p = x[y==y_i].sum(0)
 return(p+1)/(( y==y_i ).sum() +1 )
model_results = cv_model(train_set, train_labels, RidgeClassifierCV() , 'RIDGE', model_results )<find_best_params>
r = np.squeeze(np.asarray(np.log(pr(1, y=y_train, x=train_term_doc_tfidf)/pr(0, y=y_train, x=train_term_doc_tfidf))))
for n in [5, 10, 20]:
print(f'
KNN with {n} neighbors
')
 model_results = cv_model(train_set, train_labels,
KNeighborsClassifier(n_neighbors = n),
f'knn-{n}', model_results )<choose_model_class>
preds =(valid_term_doc_tfidf @ r + b)> 0
model_results = cv_model(train_set, train_labels,
ExtraTreesClassifier(n_estimators = 100, random_state = 10),
'EXT', model_results )<compute_train_metric>
print(f'Validation accuracy: {(preds == y_valid ).mean() }')
 print(f'Validation F1 score: {f1_score(y_valid, preds)}' )
model_results = cv_model(train_set, train_labels,
RandomForestClassifier(n_estimators = 100, random_state = 10),
'RF', model_results )<find_best_params>
nb_train = train_term_doc_tfidf.multiply(r)
 nb_valid = valid_term_doc_tfidf.multiply(r)
 nb_train.shape, nb_valid.shape
models = model_results['model']<filter>
model = LogisticRegression(C=4, solver='liblinear')
 model.fit(nb_train, y_train)
 preds = model.predict(nb_valid )
test_ids = list(final.loc[final['Target'].isnull() ,'idhogar'] )<predict_on_test>
print(f'Validation accuracy: {(preds == y_valid ).mean() }')
 print(f'Validation F1 score: {f1_score(y_valid, preds)}' )
def submit(model,train,train_labels,test,test_ids):
model.fit(train,train_labels)
 predictions = model.predict(test)
 predictions = pd.DataFrame({'idhogar':test_ids,'Target':predictions})
 submission = submission_base.merge(predictions,on='idhogar',how='left' ).drop(columns=['idhogar'])
 submission['Target'] = submission['Target'].fillna(4 ).astype(np.int8)
 return submission<save_to_csv>
train_term_doc_tfidf = vec_tfidf.fit_transform(train_df.text)
 test_term_doc_tfidf = vec_tfidf.transform(test_df.text)
 y_train = train_df.target.values
rf_submission = submit(RandomForestClassifier(n_estimators = 100,
random_state=10, n_jobs = -1),
train_set, train_labels, test_set, test_ids)
 rf_submission.to_csv('rf_submission.csv', index = False )<create_dataframe>
r = np.squeeze(np.asarray(np.log(pr(1, y=y_train, x=train_term_doc_tfidf)/pr(0, y=y_train, x=train_term_doc_tfidf))))
train_set = pd.DataFrame(train_set,columns=features )<data_type_conversions>
model = LogisticRegression(C=4, solver='liblinear')
 model.fit(train_term_doc_tfidf.multiply(r), y_train)
 preds = model.predict(test_term_doc_tfidf.multiply(r))
corr_matrix = train_set.corr()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape),k=1 ).astype(np.bool))
to_drop = [column for column in upper.columns if any(abs(upper[column])>0.95)]
to_drop<concatenate>
submit = pd.read_csv(path/'sample_submission.csv' )
test_set = pd.DataFrame(test_set,columns=features)
 print(test_set.shape)
 train_set, test_set = train_set.align(test_set,axis=1,join='inner')
 features = list(train_set.columns)
 print(test_set.shape )<choose_model_class>
assert all(submit.id == test_df.id)
 assert len(submit)== len(test_df)== len(preds )
estimator = RandomForestClassifier(random_state=10,n_estimators=100,n_jobs=-1)
 selector = RFECV(estimator,step=1,cv=5,scoring=scorer,n_jobs=-1 )<train_model>
submit.target = preds
<prepare_output><EOS> 
submit.to_csv('submission_060320.csv', index=False )