Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<save_to_csv>
df.to_csv('./processed.csv' )<drop_column>
!pip install pyenchant
! apt-get update
! apt-get install libenchant-dev -y
df.drop(columns=['SQBescolari', 'SQBage', 'SQBhogar_total', 'SQBedjefe', 'SQBhogar_nin', 'SQBovercrowding', 'SQBdependency', 'SQBmeaned'], inplace = True )<prepare_x_and_y>
import numpy as np
import pandas as pd
import time
from datetime import datetime
from sklearn.model_selection import train_test_split
from google.cloud import storage
from google.cloud import automl_v1beta1 as automl
from automlwrapper import AutoMLWrapper
from fastai.text import *
import spacy
import pandas as pd
import numpy as np
import re
import nltk
from nltk import word_tokenize
from tqdm import tqdm
import enchant
from nltk.metrics import edit_distance
features = [col for col in training_df.columns if col != 'Target']
X, y = training_df[features], training_df['Target']<drop_column>
replacement_patterns = [ 
(r'won't', 'will not'), 
(r'can't', 'cannot'), 
(r'i'm', 'i am'), 
(r'ain't', 'is not'), 
(r'(\w+)'ll', '\g<1> will'), 
(r'(\w+)n't', '\g<1> not'), 
(r'(\w+)'ve', '\g<1> have'), 
(r'(\w+)'s', '\g<1> is'), 
(r'(\w+)'re', '\g<1> are'), 
(r'(\w+)'d', '\g<1> would'),
]
class RegexpReplacer(object):
def __init__(self, patterns=replacement_patterns):
self.patterns = [(re.compile(regex), repl)for(regex, repl)in patterns]
def replace(self, text):
s = text
for(pattern, repl)in self.patterns:
s = re.sub(pattern, repl, s)
 return s
class SpellingReplacer(object):

 def __init__(self, dict_name='en', max_dist=2):
self.spell_dict = enchant.Dict(dict_name)
 self.max_dist = max_dist
def replace(self, word):
if self.spell_dict.check(word):
return word
suggestions = self.spell_dict.suggest(word)
 if suggestions and edit_distance(word, suggestions[0])<= self.max_dist:
return suggestions[0]
else:
return word
test_df.drop(columns = 'Target', inplace = True )<import_modules>
input_path = '/kaggle/input/'
output_path = '/kaggle/working/'
from sklearn.metrics import f1_score, make_scorer, confusion_matrix
from sklearn.model_selection import cross_val_score, cross_val_predict
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import SGDClassifier
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, VotingClassifier<train_model>
ids_with_target_error = [328,443,513,2619,3640,3900,4342,5781,6552,6554,6570,6701,6702,6729,6861,7226]
train_orig = pd.read_csv(input_path+'nlp-getting-started/train.csv', encoding = 'latin-1')
 incorrect = train_orig[train_orig['id'].isin(ids_with_target_error)]
tree = DecisionTreeClassifier(max_features= 75, class_weight='balanced')
 tree.fit(X,y )<compute_train_metric>
df_train = pd.read_csv(input_path+'disasters-on-social-media/socialmedia-disaster-tweets-DFE.csv', encoding = 'latin-1')
 df_train = df_train[['_unit_id','keyword','location','text','choose_one']]
incorrect_ids = incorrect.merge(df_train,on=['text'])['_unit_id']
df_train.rename(columns = {'_unit_id': 'id'}, inplace = True)
 df_train['target'] = df_train['choose_one'].map({'Relevant': 1, 'Not Relevant': 0})
 df_train.drop(['choose_one'], axis = 1, inplace = True)
 df_train = df_train[-df_train['target'].isna() ]
df_train = df_train.astype({'target': 'int64'})
 df_train.loc[df_train['id'].isin(incorrect_ids),'target'] = 0
df_train.head()
model = RandomForestClassifier(n_estimators=100, random_state=10, max_features= 75, n_jobs = -1 ,class_weight= 'balanced')
 cv_score = cross_val_score(model, X, y, cv = 10, scoring = 'f1_macro')
 cv_score.mean()<import_modules>
df_test = pd.read_csv(input_path+'nlp-getting-started/test.csv')
 df_test.head()
from sklearn.ensemble import GradientBoostingClassifier, VotingClassifier 
<choose_model_class>
cols = ['target','id','date','flag','user','text']
train_add = pd.read_csv(input_path+'disastertweetsinput/train_clean_add.csv',names=cols,encoding = 'latin-1', skiprows=1)
 train_add.head()
ets = []
for i in range(10):
rf = RandomForestClassifier(random_state=217+i, n_jobs=4, n_estimators=700, min_impurity_decrease=1e-3, min_samples_leaf=2, verbose=0, class_weight= 'balanced')
 ets.append(( 'rf{}'.format(i), rf))<choose_model_class>
train_add = df_train[['id','text','target']].append(train_add[['id','text','target']] )
vclf = VotingClassifier(ets, voting= 'soft' )<compute_train_metric>
def clean_tweet(text):
text = re.sub(r'http\S+', '', text)
 replacer = RegexpReplacer()
text = replacer.replace(text)
 text = re.sub(r'\b', ' ', text)
 text = re.sub(r'-', ' ', text)
 tokenizer = nltk.RegexpTokenizer(r"\w+")
 tokens = tokenizer.tokenize(text)
 replacer = SpellingReplacer()
tokens = [replacer.replace(t)for t in tokens]
wnl = nltk.WordNetLemmatizer()
tokens = [wnl.lemmatize(t)for t in tokens]
porter = nltk.PorterStemmer()
tokens = [porter.stem(t)for t in tokens]
text = ' '.join(tokens)
 return(text )
cv_score = cross_val_score(vclf, X, y, cv= 5, scoring = 'f1_macro' )<predict_on_test>
tweets = df_test['text']
tweets_cleaned = tweets.progress_apply(clean_tweet)
 df_test['text_clean'] = tweets_cleaned
cv_predict = cross_val_predict(vclf, X, y, cv = 5 )<compute_test_metric>
df = df_test[['text_clean']].append(df_train[['text_clean']])
 df = df.drop_duplicates().reset_index() ['text_clean']
df.head()
confusion_matrix(y, cv_predict )<compute_train_metric>
train = []
tokenizer = Tokenizer()
tok = SpacyTokenizer('en')
 for line in tqdm(df_train.text_clean):
lne = ' '.join(tokenizer.process_text(line, tok))
train.append(lne)
 df_train['text_tokens'] = train
test = []
tokenizer = Tokenizer()
tok = SpacyTokenizer('en')
 for line in tqdm(df_test.text_clean):
lne = ' '.join(tokenizer.process_text(line, tok))
test.append(lne)
 df_test['text_tokens'] = test
vclf = VotingClassifier(ets, voting= 'hard')
 cv_score = cross_val_score(vclf, X, y, cv = 5, scoring = 'f1_macro')
 cv_score<predict_on_test>
PROJECT_ID = 'kaggle-tweets-0234'
bucket_name = 'kaggle-tweets-0234-lcm'
region = 'us-central1'
dataset_display_name = 'disaster_tweets'
model_display_name = 'disaster_tweets_model1'
storage_client = storage.Client(project=PROJECT_ID)
 client = automl.AutoMlClient()
vclf.fit(X,y)
 vclf_hardvoting = vclf.predict(test_df )<import_modules>
project_location = client.location_path(PROJECT_ID, 'us-central1')
 response = client.list_models(project_location, '')
 print('List of models:')
 for model in response:
if model.deployment_state == \
automl.enums.Model.DeploymentState.DEPLOYED:
deployment_state = 'deployed'
else:
deployment_state = 'undeployed'
print(u'Model name: {}'.format(model.name))
print(u'Model id: {}'.format(model.name.split('/')[-1]))
print(u'Model display name: {}'.format(model.display_name))
print(u'Model create time:')
 print(u'\tseconds: {}'.format(model.create_time.seconds))
print(u'\tnanos: {}'.format(model.create_time.nanos))
print(u'Model deployment state: {}'.format(deployment_state))
import lightgbm as lgb<create_dataframe>
nlp_train_df = df_train
nlp_test_df = df_test
def callback(operation_future):
result = operation_future.result()
submit=pd.DataFrame({'Id': df['Id'][df['Target'].isna() ] , 'Target': vclf_hardvoting.astype(int)} )<count_values>
def upload_blob(bucket_name, source_file_name, destination_blob_name):

 bucket = storage_client.get_bucket(bucket_name)
 blob = bucket.blob(destination_blob_name)
 blob.upload_from_filename(source_file_name)
 print('File {} uploaded to {}'.format(
 source_file_name,
'gs://' + bucket_name + '/' + destination_blob_name))
def download_to_kaggle(bucket_name,destination_directory,file_name,prefix=None):

 os.makedirs(destination_directory, exist_ok = True)
 full_file_path = os.path.join(destination_directory, file_name)
 blobs = storage_client.list_blobs(bucket_name,prefix=prefix)
 for blob in blobs:
blob.download_to_filename(full_file_path )
submit['Target'].value_counts(normalize = True )<save_to_csv>
bucket = storage.Bucket(storage_client, name=bucket_name)
 if not bucket.exists() :
bucket.create(location=region )
submit.to_csv('./submission.csv', index= False )<define_variables>
nlp_train_df[['text_tokens','target']].to_csv('train.csv', index=False, header=False )
<load_from_csv>
training_gcs_path = 'uploads/kaggle_getstarted/full_train.csv'
train = pd.read_csv('.. /input/jigsaw-train-translated-yandex-api/train_yandex.csv')
 val = pd.read_csv('.. /input/jigsaw-multilingual-toxic-comment-classification/validation.csv', usecols=['comment_text', 'toxic', 'lang'] )<load_from_csv>
amw = AutoMLWrapper(client=client,
project_id=PROJECT_ID,
bucket_name=bucket_name,
region='us-central1',
dataset_display_name=dataset_display_name,
model_display_name=model_display_name )
test = pd.read_csv('.. /input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 test['comment_text'] = test['content']
test['toxic'] = 0.5<split>
if not amw.get_dataset_by_display_name(dataset_display_name):
print('dataset not found')
 amw.create_dataset()
amw.import_gcs_data(training_gcs_path)
 amw.dataset
[train_v, val] = train_test_split(val, test_size = 0.05, random_state = 411 )<string_transform>
if not amw.get_model_by_display_name() :
amw.train_model()
amw.model
re_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')
 def tokenize(s):
return re_tok.sub(r' \1 ', s ).split()<define_variables>
amw.model_full_path
COMMENT = 'comment_text'
LABEL = 'toxic'<define_variables>
prediction_client = automl.PredictionServiceClient()
amw.set_prediction_client(prediction_client)
 predictions_df = amw.get_predictions(nlp_test_df,
input_col_name='text_tokens',
limit=None,
threshold=0.5,
verbose=False)
binary = False<categorify>
submission_df = pd.concat([nlp_test_df['id'], predictions_df['class']], axis=1)
 submission_df.head()
vec_train = TfidfVectorizer(ngram_range=(1,1), tokenizer=tokenize,
min_df=2, max_df=0.9, strip_accents='unicode', use_idf=1, binary=binary,
smooth_idf=1, sublinear_tf=1)
 vec_train.fit(train[COMMENT])
 val_on_train = vec_train.transform(val[COMMENT])
 trn_on_train = vec_train.transform(train[COMMENT] )<categorify>
submission_df = submission_df.rename(columns={'class':'target'})
 submission_df.head()
vec_val = TfidfVectorizer(ngram_range=(1,1), tokenizer=tokenize,
min_df=2, max_df=0.9, strip_accents='unicode', use_idf=1, binary=binary,
smooth_idf=1, sublinear_tf=1)
 vec_val.fit(pd.concat([train_v[COMMENT], val[COMMENT]]))
trn_on_val = vec_val.transform(train_v[COMMENT])
 val_on_val = vec_val.transform(val[COMMENT] )<prepare_x_and_y>
submission_df.to_csv("submission.csv", index=False, header=True )