|
from nltk.tokenize import wordpunct_tokenize as word_tokenize |
|
from nltk.tokenize import sent_tokenize |
|
|
|
import re |
|
import six |
|
import textwrap |
|
|
|
_whitelist = r"[0-9a-z\,\.\/\<\>]+" |
|
_regex = "0-9a-z\,\.\/\<\>" |
|
|
|
|
|
def filter_by_lang_regex(text, ratio=0.7, regex="0-9a-z\,\.\/\<\>"): |
|
candidate_text = re.sub(r"[^" + regex + "]+", " ", six.ensure_str(text), flags=re.IGNORECASE).replace(" ", "") |
|
text = text.replace(" ", "") |
|
|
|
return (len(candidate_text) / len(text)) > ratio |
|
|
|
|
|
def filter_by_num_tokens(text, gt=64): |
|
return len(word_tokenize(text)) > gt |
|
|
|
|
|
def filter_by_num_sents(text, gt=2): |
|
return len(sent_tokenize(text)) > gt |
|
|
|
|
|
def filter_by_steps(text): |
|
return re.search('(step|mix all)', text, re.IGNORECASE) is not None |
|
|
|
|
|
def filter_by_length(text, gt=40): |
|
return len(text) > gt |
|
|
|
|
|
def filter_by_item(item_list, gt=4): |
|
return len(item_list) > gt |
|
|
|
|
|
def chars_to_preserve(sentence, whitelist): |
|
try: |
|
tokenized = re.findall(whitelist, sentence, re.IGNORECASE) |
|
return " ".join(tokenized) |
|
except Exception as error: |
|
print( |
|
textwrap.dedent( |
|
f""" |
|
Bad characters range {whitelist}, |
|
{error} |
|
""" |
|
) |
|
) |
|
raise |
|
|
|
|
|
def normalizer(text, whitelist=r"[0-9a-z\,\.\/\<\>]+", do_lowercase=False): |
|
if do_lowercase: |
|
text = text.lower() |
|
|
|
text = chars_to_preserve(text, whitelist=whitelist) |
|
text = " ".join([word.strip() for word in text.split() if word.strip()]) |
|
text = text.strip() |
|
|
|
return text |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
