utils.py

import re
import string
from collections.abc import Iterable

import nltk
import numpy as np
from keras.preprocessing.text import Tokenizer
from keras.utils import pad_sequences
from nltk.corpus import stopwords

nltk.download('stopwords')


# Apply regex and do cleaning.
def clean_text(words: str) -> str:
    words = str(words).lower()
    words = re.sub('\[.*?\]', '', words)
    words = re.sub('https?://\S+|www\.\S+', '', words)
    words = re.sub('<.*?>+', '', words)
    words = re.sub(r'@\w+', '', words)
    words = re.sub('[%s]' % re.escape(string.punctuation), '', words)
    words = re.sub('\n', '', words)
    words = re.sub('\w*\d\w*', '', words)

    stopword = set(stopwords.words('english'))
    words = ' '.join(
        [word for word in words.split(' ') if word not in stopword])

    stemmer = nltk.SnowballStemmer("english")
    words = ' '.join([stemmer.stem(word) for word in words.split(' ')])

    return words


def tokenize_and_pad(text_list: Iterable[str], tokenizer: Tokenizer, max_len: int) -> np.ndarray[np.str_]:
    sequences = tokenizer.texts_to_sequences(text_list)
    sequences_matrix = pad_sequences(sequences, maxlen=max_len)
    return sequences_matrix