Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<define_variables>
import warnings, re, string
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
from nltk.tokenize import word_tokenize
from nltk.stem import SnowballStemmer
from nltk.corpus import stopwords
from sklearn.linear_model import PassiveAggressiveClassifier, LogisticRegression, SGDClassifier
from sklearn.metrics import precision_score, r2_score, recall_score
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import cross_val_score, KFold
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline
from sklearn.svm import SVC
from wordcloud import WordCloud
n_steps = NEXAMPLESPEREPOCH // BATCH_SIZE
num_nen=0
if use_validation_during_pretraining:
num_nen+=total_nen_train_toxic+total_nen_train_non_toxic
if use_dann:
num_nen += total_nen_unknown
n_steps,num_nen<categorify>
warnings.filterwarnings('ignore')
 pd.set_option('max_colwidth', -1)
 Colors = {'Reds':'
Etq = ['Not Disaster', 'Disaster']
PATH_DATA = '.. /input/nlp-getting-started/'
num_dataset_section=2
if use_translated_data:
num_dataset_section+=2
def f0() :
return unknown_nen_dataset().repeat().take(total_nen_unknown)
 def f1() :
return toxic_train_en_dataset().repeat().take(int(( NEXAMPLESPEREPOCH-num_nen)/num_dataset_section))
def f2() :
return non_toxic_train_en_dataset().repeat().take(int(( NEXAMPLESPEREPOCH-num_nen)/num_dataset_section))
def f3() :
return toxic_train_translated_dataset().repeat().take(int(( NEXAMPLESPEREPOCH-num_nen)/num_dataset_section))
def f4() :
return non_toxic_train_translated_dataset().repeat().take(int(( NEXAMPLESPEREPOCH-num_nen)/num_dataset_section))
def f5() :
return toxic_train_nen_dataset()
def f6() :
return non_toxic_train_nen_dataset()
branches={0:f0,1:f1,2:f2,3:f3,4:f4,5:f5,6:f6}
mixedDatasetParams=[1,2]
if use_dann:
mixedDatasetParams.append(0)
 if use_validation_during_pretraining:
mixedDatasetParams.extend([5,6])
 if use_translated_data:
mixedDatasetParams.extend([3,4])
 def mapParamsToDataset(params):
return tf.switch_case(params,branches)
<prepare_x_and_y>
df =(pd.read_csv(f'{PATH_DATA}train.csv'
, sep = ','
, header = 0)
.sample(frac = 1, random_state = 0))
df.head()
cycle_length=4
if use_dann:
cycle_length+=1
def non_parallel_dataset() :
return tf.data.Dataset.from_tensor_slices(mixedDatasetParams ).interleave(mapParamsToDataset, num_parallel_calls=AUTO,cycle_length=cycle_length ).take(NEXAMPLESPEREPOCH)
 if interativeEnvironment:
train_dict_dataset=non_parallel_dataset()
train_dict_dataset<create_dataframe>
def clean_df(df):
df.text = df.text.apply(lambda x: x.lower())
 df.text = df.text.apply(lambda x: re.sub(r'https?://\S+|www\.\S+', '', x))
df.text = df.text.apply(lambda x: re.sub(r'\[.*?\]', '', x))
df.text = df.text.apply(lambda x: re.sub(r'<.*?>', '', x))
df.text = df.text.apply(lambda x: re.sub(r'[%s]' % re.escape(string.punctuation), '', x))
df.text = df.text.apply(lambda x: re.sub(r'
', '', x))
df.text = df.text.apply(lambda x: re.sub(r'\w*\d\w*', '', x))
return df
df = clean_df(df)
 df.head()
non_parallel_dataset()<init_hyperparams>
Punct_List = dict(( ord(punct), None)for punct in string.punctuation)
 Stemmer_List = SnowballStemmer('english')
 def TxNormalize(text):
tokens = word_tokenize(str(text ).replace('/',' ' ).translate(Punct_List))
return [x for x in tokens if x not in stopwords.words('english')]
def Crear_WordCloud(ax, tokens, Color, Titulo, Theme):
if len(tokens)> 0:
wc = WordCloud(width = 6000
, height = 3500
, min_font_size = 60
, max_words = 100
, background_color = 'white'
, colormap = Theme
, random_state = 0
).generate(tokens)
 ax.imshow(wc)
 ax.set_title(Titulo, fontsize = 60, color = Color, y = 1.03)
 ax.axis('off')
 def Plot(ax, df, Color, Titulo):
datos = df.word.value_counts(sort = True ).nlargest(25)
 ax.barh(datos.index, datos.values, color = Color)
 ax.tick_params(left = False, bottom = False)
 ax.invert_yaxis()
[spine.set_visible(False)for spine in ax.spines.values() ]
ax.spines['left'].set_visible(True)
 ax.set_title(Titulo.capitalize() , fontsize = 14)
 ax.grid(axis = 'x', alpha = 0.25)
 def tokenize(df, filtro):
tokens = []
for i in df[df.target == filtro].text:
tokens += TxNormalize(i)
 return tokens
%%time
if interativeEnvironment:
dlist=train_dict_dataset.repeat()
epoch_metrics={}
shard_metrics={}
for i,l in tqdm(enumerate(dlist)) :
epoch=str(i//NEXAMPLESPEREPOCH)
 shard=str(i//40000)
 if epoch=="10":
break
if epoch in epoch_metrics.keys() :
pass
else:
total_by_source=Counter()
total_by_toxic=Counter()
total_by_toxic_source=Counter()
total_by_s_toxic=Counter()
total=0
epoch_metrics[epoch]={"by_toxic":total_by_toxic,"by_source":total_by_source,"total":total,"by_both":total_by_toxic_source,"by_s_toxic":total_by_s_toxic}
if shard in shard_metrics.keys() :
pass
else:
stotal_by_source=Counter()
stotal_by_toxic=Counter()
stotal_by_toxic_source=Counter()
stotal_by_s_toxic=Counter()
stotal=0
shard_metrics[shard]={"by_toxic":stotal_by_toxic,"by_source":stotal_by_source,"total":stotal,"by_both":stotal_by_toxic_source,"by_s_toxic":stotal_by_s_toxic}
total+=1
total_by_source.update([l["source"].numpy() ])
 total_by_toxic.update([-1 if l["toxic"] ==-1 else 1 if l["toxic"].numpy() >=0.5 else 0])
 total_by_s_toxic.update([-1 if l["s_toxic"] ==-1 else 1 if l["s_toxic"].numpy() >=0.5 else 0])
 total_by_toxic_source.update([(round(l["toxic"].numpy()),l["source"].numpy())])
 stotal+=1
stotal_by_source.update([l["source"].numpy() ])
 stotal_by_toxic.update([-1 if l["toxic"] ==-1 else 1 if l["toxic"].numpy() >=0.5 else 0])
 stotal_by_s_toxic.update([-1 if l["s_toxic"] ==-1 else 1 if l["s_toxic"].numpy() >=0.5 else 0])
 stotal_by_toxic_source.update([(round(l["toxic"].numpy()),l["source"].numpy())])
 for epoch,epoch_metric in epoch_metrics.items() :
print("Epoch:-",epoch,"Toxic:-",epoch_metric["by_toxic"].most_common() ,"By SToxic:",epoch_metric["by_s_toxic"].most_common() ,"By source:-",epoch_metric["by_source"].most_common() ,"
" )<init_hyperparams>
Words = [ 
(df[(df.target == i)].text 
.apply(TxNormalize)
.explode() 
.str.cat(sep = ' '))
for i in range(2)]
if interativeEnvironment:
for epoch,epoch_metric in shard_metrics.items() :
print("Shard:-",epoch,"Toxic:-",epoch_metric["by_toxic"].most_common() ,"Stoxic:",epoch_metric["by_s_toxic"].most_common() ,"By source:-",epoch_metric["by_source"].most_common() ,"
" )<load_pretrained>
Params = {'Regresión Logística': LogisticRegression(C = 1e5
, class_weight = 'balanced'
, random_state = 0)
 , 'Pasive Aggressive Classifier': PassiveAggressiveClassifier(max_iter = 50
, random_state = 0)
 , 'SGD': SGDClassifier(max_iter = 5
, tol = None
, random_state = 0)
 , 'Naive Bayes': MultinomialNB(alpha = 1)
 , 'Support Vector Machine': SVC(kernel = 'linear'
, C = 1
, gamma = 'scale'
, random_state = 0)}
def Modelo(Clf, X, y):
clf = Pipeline([ 
('tfidf', TfidfVectorizer(ngram_range =(1, 4)
 , analyzer = 'word'
, stop_words = 'english'))
,('Classifier', Clf)
 ])
 return clf.fit(X, y )
def opus_files(lang):
return tf.io.gfile.glob(OPUS_DS_PATH + "/{}_*.tfrecord".format(lang)+extension)
 opus_files("es" )<data_type_conversions>
X = df.text
y = df.target
Result = {}
kf = KFold(n_splits = 10)
 for i in Params:
Score = []
clf = Modelo(Params[i], X, y)
 Score.append(Params[i].__class__.__name__)
 Score.append(cross_val_score(clf, X, y, cv = kf, scoring = 'accuracy' ).mean())
 Result[i] = Score
Models =(pd.DataFrame(Result, index=['Tipo', 'Cross Val'])
.transpose() 
.sort_values(by = 'Cross Val', ascending = False)
.reset_index() 
.rename(columns = {'index':'Modelo'}))
Models
if use_parcor:
parallel_feature_description = {
'tokens_en': tf.io.FixedLenFeature([192], tf.int64),
'en_attn_mask': tf.io.FixedLenFeature([192], tf.int64),
'tokens_nen': tf.io.FixedLenFeature([192], tf.int64),
'nen_attn_mask': tf.io.FixedLenFeature([192], tf.int64),
'lang':tf.io.FixedLenFeature([], tf.string),
'source':tf.io.FixedLenFeature([], tf.int64,default_value=[5]),
's_toxic':tf.io.FixedLenFeature([2], tf.float32,default_value=[-1.0,-1.0]),
'is_same':tf.io.FixedLenFeature([], tf.float32,default_value=[1.0]),
}
def _parse_parallel_proto(example_proto):
example= tf.io.parse_single_example(example_proto, parallel_feature_description)
 return example
if IN_COLAB:
compression_type="GZIP"
else:
compression_type=""
def perFileParallelDataset(filename,feature_description=feature_description):
tf.print("Retrieving from filename",filename)
 return tf.data.TFRecordDataset(filename,compression_type=compression_type ).map(_parse_parallel_proto)
 def make_parallel_pairs(example):
ret={}
ret["tokens"]=tf.stack([example["tokens_en"],example["tokens_nen"]])
 ret["mask"]=tf.stack([example["en_attn_mask"],example["nen_attn_mask"]])
 ret["s_toxic"]=example["s_toxic"]
ret["is_same"]=example["is_same"]
ret["lang"]=example["lang"]
return ret
def parallel_dataset(files):
return tf.data.Dataset.from_tensor_slices(files ).shuffle(2048 ).interleave(perFileParallelDataset ).map(make_parallel_pairs)
 def parallel_labelled_dataset(labelled_dataset):
return labelled_dataset.batch(2 ).map(make_parallel_labelled_pairs)
 def make_parallel_labelled_pairs(example):
example["is_same"]=tf.constant(0.0)
 return example
langs=["es","fr","pt","tr","ru","it"]
langFiles=list(map(opus_files,langs))
def all_parallel_dataset() :
return tf.data.Dataset.from_tensor_slices(langFiles ).interleave(parallel_dataset, num_parallel_calls=AUTO,cycle_length=len(langs))
def parallel_nonparallel_mixer(params):
return tf.cond(params=="parallel",lambda :all_parallel_dataset().take(int(NEXAMPLESPEREPOCH/4)).map(_form_tuple),lambda :parallel_labelled_dataset(non_parallel_dataset() ).take(int(NEXAMPLESPEREPOCH/4)).map(_form_tuple))
def parcor_dataset() :
return tf.data.Dataset.from_tensor_slices(["parallel","nonparallel"] ).interleave(parallel_nonparallel_mixer,num_parallel_calls=2 )<concatenate>
df =(pd.read_csv(f'{PATH_DATA}test.csv'
, sep = ','
, header = 0))
df = clean_df(df)
 df.head()
if interativeEnvironment:
ds=list(parallel_labelled_dataset(valid_nen_dataset() ).take(7))
print(list(map(lambda a:a["is_same"],ds)))
<categorify>
Best = Modelo(Params[Models['Modelo'][0]], X, y)
 df['target'] = Best.predict(df.text )
<import_modules><EOS> 
df[['id','target']].to_csv('submission.csv', index = False )