Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<save_to_csv>
!pip install tweet-preprocessor
rf5 = RandomForestClassifier(n_jobs=-1, n_estimators=200, random_state=42, oob_score=True, class_weight='balanced')
 rf5.fit(train[feats], train['Target'])
 test['Target'] = rf5.predict(test[feats] ).astype(int)
 test[['Id', 'Target']].to_csv('submission.csv', index=False)
<load_from_csv>
import os
from google.cloud import storage, automl_v1beta1 as automl
import numpy as np
import pandas as pd
from sklearn import feature_extraction, linear_model, model_selection, preprocessing
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
import scipy as sp
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold, KFold, GridSearchCV
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score, confusion_matrix, f1_score
from sklearn.naive_bayes import GaussianNB, MultinomialNB
import seaborn as sns
import matplotlib.pyplot as plt
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
from nltk.stem import WordNetLemmatizer
import nltk
from collections import Counter
import string
import re
from nltk.corpus import wordnet as wn
from statistics import mean
import preprocessor
df = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 test = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/test.csv')
 df.shape, test.shape<concatenate>
from sklearn import metrics
df_all = df.append(test)
 df_all.shape<data_type_conversions>
stop = set(STOPWORDS ).union(set(['FAV' , 'RT']))
lemma = WordNetLemmatizer()
preprocessor.set_options(preprocessor.OPT.URL, preprocessor.OPT.MENTION, preprocessor.OPT.NUMBER, preprocessor.OPT.RESERVED)
 def clean(text):
text = preprocessor.clean(text)
 text = re.sub(r'[^\w\s]','',text)
 stop_free = " ".join([i for i in text.split(' ')if(i not in stop)])
 normalized = " ".join(lemma.lemmatize(word)for word in stop_free.split())
 return normalized
df_all['edjefa'] = df_all['edjefa'].replace(mapeamento ).astype(int)
 df_all['edjefe'] = df_all['edjefe'].replace(mapeamento ).astype(int)
 df_all['dependency'] = df_all['dependency'].replace(mapeamento ).astype(float )<data_type_conversions>
train_df = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test_df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv" )
df_all['v2a1'].fillna(-1, inplace=True)
 df_all['v18q1'].fillna(0, inplace=True)
 df_all['SQBmeaned'].fillna(-1, inplace=True)
 df_all['meaneduc'].fillna(-1, inplace=True)
 df_all['rez_esc'].fillna(-1, inplace=True )<define_variables>
train_df.text = train_df.text.apply(clean)
 test_df.text = test_df.text.apply(clean )
feats = [c for c in df_all.columns if c not in ['Id', 'idhogar', 'Target']]<save_to_csv>
PROJECT_ID = 'automl-kaggle-263107'
rf6 = RandomForestClassifier(n_jobs=-1, n_estimators=200, random_state=42, oob_score=True, class_weight='balanced')
 rf6.fit(train[feats], train['Target'])
 test['Target'] = rf6.predict(test[feats] ).astype(int)
 test[['Id', 'Target']].to_csv('submission.csv', index=False )<save_to_csv>
BUCKET_NAME = 'automl-disaster-tweet-cleaned'
BUCKET_REGION = 'us-central1'
rf7 = RandomForestClassifier(max_depth=None, random_state=42, n_jobs=4, n_estimators=700,
min_impurity_decrease=1e-3, min_samples_leaf=2,
verbose=0, class_weight='balanced')
 rf7.fit(train[feats], train['Target'])
 test['Target'] = rf7.predict(test[feats] ).astype(int)
 test[['Id', 'Target']].to_csv('submission.csv', index=False )<import_modules>
storage_client = storage.Client(project=PROJECT_ID)
 tables_gcs_client = automl.GcsClient(client=storage_client, bucket_name=BUCKET_NAME)
 automl_client = automl.AutoMlClient()
prediction_client = automl.PredictionServiceClient()
tables_client = automl.TablesClient(project=PROJECT_ID, region=BUCKET_REGION, client=automl_client, gcs_client=tables_gcs_client, prediction_client=prediction_client )
import numpy as np
import pandas as pd<load_from_csv>
bucket = storage.Bucket(storage_client, name=BUCKET_NAME)
 if not bucket.exists() :
bucket.create(location=BUCKET_REGION )
train = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 test = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/test.csv')
 print('Tamanho dos datasets Train e Test:')
 print(' - Train - Linhas:', train.shape[0],'Colunas:', train.shape[1],'
- Test - Linhas:', test.shape[0],'Colunas:', test.shape[1] )<concatenate>
def upload_blob(bucket_name, source_file_name, destination_blob_name):

 bucket = storage_client.get_bucket(bucket_name)
 blob = bucket.blob(destination_blob_name)
 blob.upload_from_filename(source_file_name)
 print('File {} uploaded to {}.'.format(
 source_file_name,
destination_blob_name))
def download_to_kaggle(bucket_name,destination_directory,file_name,prefix=None):

 os.makedirs(destination_directory, exist_ok = True)
 full_file_path = os.path.join(destination_directory, file_name)
 blobs = storage_client.list_blobs(bucket_name,prefix=prefix)
 for blob in blobs:
blob.download_to_filename(full_file_path )
df_all = train.append(test)
 print('Informações do novo dataset:
df_all - Linhas:', df_all.shape[0],'Colunas:', df_all.shape[1] )<count_values>
train_df[['id','text','target']].to_csv('/kaggle/working/train.csv', index=False)
 test_df[['id','text']].to_csv('/kaggle/working/test.csv', index=False )
df_all['edjefa'].value_counts()<count_values>
upload_blob(BUCKET_NAME, '/kaggle/working/train.csv', 'train.csv')
 upload_blob(BUCKET_NAME, '/kaggle/working/test.csv', 'test.csv' )
df_all['edjefe'].value_counts()<count_values>
dataset_display_name = 'tweet_disaster_cleaned'
new_dataset = False
try:
dataset = tables_client.get_dataset(dataset_display_name=dataset_display_name)
 except:
new_dataset = True
dataset = tables_client.create_dataset(dataset_display_name )
df_all['dependency'].value_counts()<define_variables>
if new_dataset:
gcs_input_uris = ['gs://' + BUCKET_NAME + '/train.csv']
import_data_operation = tables_client.import_data(
 dataset=dataset,
gcs_input_uris=gcs_input_uris 
)
 print('Dataset
import_data_operation.result()
mapeamento = {'yes': 1, 'no': 0}<data_type_conversions>
ID_COLUMN = 'id'
df_all['edjefa'] = df_all['edjefa'].replace(mapeamento ).astype(int)
 df_all['edjefe'] = df_all['edjefe'].replace(mapeamento ).astype(int )<count_values>
TRAIN_BUDGET = 1000
model = None
model_display_name = 'tweet_disaster_model_clean'
try:
model = tables_client.get_model(model_display_name=model_display_name)
 except:
response = tables_client.create_model(
 model_display_name,
dataset=dataset,
train_budget_milli_node_hours=TRAIN_BUDGET,
exclude_column_spec_names=[TARGET_COLUMN,ID_COLUMN] 
)
 print('Create model operation: {}'.format(response.operation))
model = response.result()
print(model )
df_all['dependency'].value_counts()<feature_engineering>
gcs_input_uris = 'gs://' + BUCKET_NAME + '/test.csv'
gcs_output_uri_prefix = 'gs://' + BUCKET_NAME + '/predictions'
batch_predict_response = tables_client.batch_predict(
 model=model,
gcs_input_uris=gcs_input_uris,
gcs_output_uri_prefix=gcs_output_uri_prefix, 
)
 print('Batch prediction operation: {}'.format(batch_predict_response.operation))
batch_predict_result = batch_predict_response.result()
batch_predict_response.metadata
df_all['dependency'] = np.sqrt(df_all['SQBdependency'] )<count_missing_values>
gcs_output_folder = batch_predict_response.metadata.batch_predict_details.output_info.gcs_output_directory.replace('gs://' + BUCKET_NAME + '/','')
 download_to_kaggle(BUCKET_NAME,'/kaggle/working','submissions.csv', prefix=gcs_output_folder )
df_all.isnull().sum().sort_values()<data_type_conversions>
preds_df = pd.read_csv("/kaggle/working/submissions.csv")
 preds_df = preds_df.sort_values(by=['id'])
 preds_df['target'] =(preds_df['target_1_score'] >= 0.5 ).astype(int )
df_all['v2a1'] = df_all['v2a1'].fillna(value=df_all['tipovivi3'])
 df_all['v18q1'] = df_all['v18q1'].fillna(value=df_all['v18q'])
 df_all['rez_esc'].fillna(0, inplace=True )<count_missing_values>
preds_df[['id','target']].to_csv("submission.csv", index=False )
df_all.isnull().sum().sort_values()<correct_missing_values>
tfidf_vectorizer = feature_extraction.text.TfidfVectorizer(ngram_range =(1,2), stop_words='english',strip_accents='unicode' )
df_all['Target'].fillna(-1, inplace=True )<import_modules>
train_vectors = tfidf_vectorizer.fit_transform(train_df["text"])
 test_vectors = tfidf_vectorizer.transform(test_df["text"] )
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.metrics import confusion_matrix, f1_score, make_scorer<define_variables>
feature_cols = ['keyword', 'location']
X = train_df[feature_cols]
y = train_df.target
one_hot_encoded_training_predictors = pd.get_dummies(X)
 clf = RandomForestClassifier(n_estimators = 100)
 scores = model_selection.cross_val_score(clf, one_hot_encoded_training_predictors, y, cv=5, scoring="f1")
 scores
feats = [c for c in df_all.columns if c not in ['Id', 'idhogar', 'Target']]<split>
clf.fit(one_hot_encoded_training_predictors, y )
train, test = df_all[df_all['Target'] != -1], df_all[df_all['Target'] == -1]<choose_model_class>
clf = linear_model.RidgeClassifier()
rf = RandomForestClassifier(n_jobs=-1, n_estimators=200, oob_score=True, random_state=42 )<train_model>
scores = model_selection.cross_val_score(clf, train_vectors, train_df["target"], cv=10, scoring="f1")
 scores
rf.fit(train[feats], train['Target'] )<predict_on_test>
clf.fit(train_vectors, train_df["target"] )
test['Target'] = rf.predict(test[feats] ).astype(int )<count_values>
parameters = {
'gamma': [0.7, 1, 'auto', 'scale']
}
clf = GridSearchCV(SVC(kernel='rbf'), parameters, cv=5, n_jobs=-1, scoring="f1" ).fit(train_vectors, train_df["target"] )
test['Target'].value_counts(normalize=True )<save_to_csv>
clf.best_estimator_
<feature_engineering>
clf.best_score_
df_all['hsize-pc'] = df_all['hhsize'] / df_all['tamviv']
df_all['phone-pc'] = df_all['qmobilephone'] / df_all['tamviv']
df_all['tablets-pc'] = df_all['v18q1'] / df_all['tamviv']
df_all['rooms-pc'] = df_all['rooms'] / df_all['tamviv']
df_all['rent-pc'] = df_all['v2a1'] / df_all['tamviv']<split>
sample_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 sample_submission["target"] = clf.predict(test_vectors)
 df = pd.DataFrame({'text' : test_df['text'], 'prediction' : sample_submission["target"]} )
train, test = df_all[df_all['Target'] != -1], df_all[df_all['Target'] == -1]<define_variables>
sample_submission.to_csv("submission1.csv", index=False )
retira_cols = ['agesq', 'area2', 'hogar_total', 'male', 'public', 'r4t3', 'tamhog', 'tamviv',
'hhsize', 'v18q', 'v14a', 'mobilephone', 'female']
parentesco_cols = [colunas for colunas in train.columns.tolist() if 'parentesco' in colunas and 'parentesco1' not in colunas]
retira_cols.extend(parentesco_cols)
 train = train.drop(retira_cols, axis=1)
 test = test.drop(retira_cols, axis=1)
 df_all = df_all.drop(retira_cols, axis=1 )<define_variables>
print(tf.__version__)
feats = [c for c in df_all.columns if c not in ['Id', 'idhogar', 'Target']]<define_variables>
X = train_df["text"]
y = train_df["target"]
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.3, random_state=42 )
heads = train[train['parentesco1'] == 1]<choose_model_class>
vocab_size = 10000
embedding_dim = 16
max_length = 30
trunc_type='post'
padding_type='post'
oov_tok = "<OOV>"
rf2 = RandomForestClassifier(n_jobs=-1, n_estimators=200, oob_score=True, random_state=42 )<train_model>
tokenizer = Tokenizer(num_words = vocab_size, oov_token=oov_tok)
 tokenizer.fit_on_texts(X_train)
 word_index = tokenizer.word_index
sequences = tokenizer.texts_to_sequences(X_train)
 padded = pad_sequences(sequences,maxlen=max_length, padding=padding_type, truncating=trunc_type)
 testing_sequences = tokenizer.texts_to_sequences(X_val)
 testing_padded = pad_sequences(testing_sequences,maxlen=max_length, padding=padding_type, truncating=trunc_type )
rf2.fit(train[feats], train['Target'] )<predict_on_test>
reverse_word_index = dict([(value, key)for(key, value)in word_index.items() ])
 def decode_sentence(text):
return ' '.join([reverse_word_index.get(i, '?')for i in text])
 print(decode_sentence(padded[0]))
test['Target'] = rf2.predict(test[feats] ).astype(int )<count_values>
!wget --no-check-certificate \
https://storage.googleapis.com/laurencemoroney-blog.appspot.com/glove.6B.100d.txt \
-O /tmp/glove.6B.100d.txt
embeddings_index = {};
vocab_size=len(word_index)
 embedding_dim = 100
with open('/tmp/glove.6B.100d.txt')as f:
for line in f:
values = line.split() ;
word = values[0];
coefs = np.asarray(values[1:], dtype='float32');
embeddings_index[word] = coefs;
embeddings_matrix = np.zeros(( vocab_size+1, embedding_dim)) ;
for word, i in word_index.items() :
embedding_vector = embeddings_index.get(word);
if embedding_vector is not None:
embeddings_matrix[i] = embedding_vector;
test['Target'].value_counts(normalize=True )<save_to_csv>
model = tf.keras.Sequential([
tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),
tf.keras.layers.GlobalAveragePooling1D() ,
tf.keras.layers.Dense(24, activation='relu'),
tf.keras.layers.Dense(1, activation='sigmoid')
 ])
 model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy',tf.keras.metrics.AUC() ])
 model.summary()
test[['Id', 'Target']].to_csv('submission.csv', index=False )<filter>
num_epochs = 3
history = model.fit(padded, y_train, epochs=num_epochs, validation_data=(testing_padded, y_val))
heads2 = train[train['parentesco1'] == 1]<define_variables>
model_loss = pd.DataFrame(model.history.history)
 model_loss.head()
feats = [c for c in df_all.columns if c not in ['Id', 'idhogar', 'Target']]<choose_model_class>
testing_sequences2 = tokenizer.texts_to_sequences(test_df.text)
 testing_padded2 = pad_sequences(testing_sequences2, maxlen=max_length, padding=padding_type, truncating=trunc_type )
rf3 = RandomForestClassifier(n_jobs=-1, n_estimators=200, oob_score=True, random_state=42 )<train_model>
probabilities = model.predict(testing_padded2 )
rf3.fit(heads2[feats], heads2['Target'] )<predict_on_test>
predictions =(probabilities > 0.5 ).astype(int)
 predictions = np.ndarray.flatten(predictions)
 pd.value_counts(predictions )
test['Target'] = rf3.predict(test[feats] ).astype(int )<save_to_csv>
original_test_df = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 df = pd.DataFrame({'text' : original_test_df['text'],'cleaned_text' : test_df['text'], 'prediction' : predictions,'probabilities' : np.ndarray.flatten(probabilities)})
 df.to_csv("test_df.csv", index=False )
test[['Id', 'Target']].to_csv('submission.csv', index=False )<choose_model_class>
sample_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 sample_submission["target"] = predictions
sample_submission.to_csv("submission.csv", index=False )
rf4 = RandomForestClassifier(max_depth=None, random_state=42, n_jobs=4, n_estimators=700,
min_impurity_decrease=1e-3, min_samples_leaf=2,
verbose=0, class_weight='balanced' )<train_model>
import tensorflow_hub as hub
import lightgbm as lgb
from lightgbm import LGBMClassifier
rf4.fit(heads2[feats], heads2['Target'] )<predict_on_test>
module_url = "https://tfhub.dev/google/nnlm-en-dim128/2"
embed = hub.KerasLayer(module_url)
 embeddings = embed(["A long sentence.", "single-word",
"http://example.com"])
 print(embeddings.shape )
test['Target'] = rf4.predict(test[feats] ).astype(int )<save_to_csv>
embed = hub.load("https://tfhub.dev/google/universal-sentence-encoder/3" )
test[['Id', 'Target']].to_csv('submission.csv', index=False )<train_model>
X_train_embeddings = embed(train_df.text.values)
 X_test_embeddings = embed(test_df.text.values )
train_y = train['Target']
X_train, X_test, y_train, y_test = train_test_split(train[feats], train_y, test_size=0.2, random_state=42)
 F1_scorer = make_scorer(f1_score, greater_is_better=True, average='macro')
 gbm = lgb.LGBMClassifier(boosting_type='dart', objective='multiclassova', class_weight='balanced', random_state=0)
 gbm.fit(X_train, y_train )<train_model>
params = {
'learning_rate': 0.04,
'n_estimators': 1500,
'colsample_bytree': 0.4,
'metric':'auc'
}
train_y = train.Target
X_train, X_test, y_train, y_test = train_test_split(train[feats], train_y, test_size=0.2, random_state=42)
 F1_scorer = make_scorer(f1_score, greater_is_better=True, average='macro')
 gbm = lgb.LGBMClassifier(boosting_type='dart', objective='multiclassova', class_weight='balanced', random_state=0)
 gbm.fit(X_train, y_train )<compute_train_metric>
text_clf = LGBMClassifier(**params )
y_test_pred = gbm.predict(X_test)
 cm = confusion_matrix(y_test, y_test_pred)
 f1 = f1_score(y_test, y_test_pred, average='macro')
 print("confusion matrix:
", cm)
 print("macro F1 score:
", f1 )<load_from_csv>
text_clf.fit(X_train_embeddings['outputs'][:5000,:], train_df.target.values[:5000],
eval_set=[(X_train_embeddings['outputs'][:5000,:], train_df.target.values[:5000]), 
(X_train_embeddings['outputs'][5000:,:], train_df.target.values[5000:])],
verbose=200, early_stopping_rounds=20, 
)
df = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 df.head(10 )<import_modules>
text_clf.fit(X_train_embeddings['outputs'][:5000,:], train_df.target.values[:5000])
 Y_pred = text_clf.predict(X_train_embeddings['outputs'][5000:] )
KMeansSMOTE,SMOTE,SVMSMOTE)
 EditedNearestNeighbours,
RepeatedEditedNearestNeighbours,
NeighbourhoodCleaningRule,AllKNN,TomekLinks)
<groupby>
print(metrics.classification_report(train_df.target[5000:], Y_pred, digits=3),)
 print(metrics.confusion_matrix(train_df.target[5000:], Y_pred))
g = df.columns.to_series().groupby(df.dtypes ).groups<data_type_conversions>
text_clf.fit(X_train_embeddings['outputs'], train_df.target.values)
 pred_test = text_clf.predict(X_test_embeddings['outputs'] )
def trata_cat(df):
cols = ['dependency', 'edjefe', 'edjefa']
df[cols] = df[cols].replace({'yes':1, 'no':0})
 df["dependency"] = df["dependency"].astype(float)
 df["edjefe"] = df["edjefe"].astype(int)
 df["edjefa"] = df["edjefa"].astype(int)
 return df<concatenate>
df = pd.DataFrame({'cleaned_text' : test_df['text'], 'prediction' : pred_test})
 df.head(20 )
def dropa_trata(df):
df_limpo = df.drop(["Id","idhogar"],axis=1 ).copy()
return trata_cat(df_limpo )<train_model>
sample_submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 sample_submission["target"] = pred_test
sample_submission.to_csv("submission.csv", index=False )