File size: 1,553 Bytes
c188624 eb5e9dd c188624 eb5e9dd ec377f7 c188624 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 |
import pandas as pd
import numpy as np
import nltk
from nltk.tokenize import word_tokenize, RegexpTokenizer
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import CountVectorizer
import csv
nltk.download('stopwords')
nltk.download('punkt')
stopwords_es = stopwords.words('spanish')
def eliminar_puntuacion(articulo):
deletetion_symbols = ['!','(',')',"'",'-','[',']','{','}',';',':','"','“','’','”',"'",'`','‘','``','\\' ,'/','|',',','|','<','>','.','..','...','?','@',"#",'$','^','&','*','_','~','+','%','=','¿','¡',"''"]
new_articulo = ""
for x in articulo:
if x not in deletetion_symbols:
new_articulo += x
return new_articulo
def eliminar_stopwords(articulo):
articulo_splitted=articulo.split()
new_articulo = ""
for x in articulo_splitted:
if x not in stopwords_es:
new_articulo += " " + x
return new_articulo
def limpieza_articulos(df):
df_titulos=pd.DataFrame(df['titulo'], columns=['titulo'])
# Colocando texto en minusculas
df_titulos['titulo']= df_titulos['titulo'].apply(lambda x: x.lower())
# Eliminando signos de puntuacion
df_titulos['titulo']= df_titulos['titulo'].apply(lambda x: eliminar_puntuacion(x))
# Eliminando palabras vacias (stopwords) utilizando el corpus para estos fines que tiene nltk
df_titulos['titulo']= df_titulos['titulo'].apply(lambda x: eliminar_stopwords(x))
all_text = ' '. join(df_titulos['titulo'])
vocab= np.unique(word_tokenize(all_text))
return vocab
|