Spaces:
Running
Running
File size: 5,858 Bytes
3affa92 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 |
# -*- coding: utf-8 -*-
from __future__ import print_function, division, unicode_literals
import sys
import re
import string
import emoji
from itertools import groupby
import numpy as np
from torchmoji.tokenizer import RE_MENTION, RE_URL
from torchmoji.global_variables import SPECIAL_TOKENS
try:
unichr # Python 2
except NameError:
unichr = chr # Python 3
AtMentionRegex = re.compile(RE_MENTION)
urlRegex = re.compile(RE_URL)
# from http://bit.ly/2rdjgjE (UTF-8 encodings and Unicode chars)
VARIATION_SELECTORS = [ '\ufe00',
'\ufe01',
'\ufe02',
'\ufe03',
'\ufe04',
'\ufe05',
'\ufe06',
'\ufe07',
'\ufe08',
'\ufe09',
'\ufe0a',
'\ufe0b',
'\ufe0c',
'\ufe0d',
'\ufe0e',
'\ufe0f']
# from https://stackoverflow.com/questions/92438/stripping-non-printable-characters-from-a-string-in-python
ALL_CHARS = (unichr(i) for i in range(sys.maxunicode))
CONTROL_CHARS = ''.join(map(unichr, list(range(0,32)) + list(range(127,160))))
CONTROL_CHAR_REGEX = re.compile('[%s]' % re.escape(CONTROL_CHARS))
def is_special_token(word):
equal = False
for spec in SPECIAL_TOKENS:
if word == spec:
equal = True
break
return equal
def mostly_english(words, english, pct_eng_short=0.5, pct_eng_long=0.6, ignore_special_tokens=True, min_length=2):
""" Ensure text meets threshold for containing English words """
n_words = 0
n_english = 0
if english is None:
return True, 0, 0
for w in words:
if len(w) < min_length:
continue
if punct_word(w):
continue
if ignore_special_tokens and is_special_token(w):
continue
n_words += 1
if w in english:
n_english += 1
if n_words < 2:
return True, n_words, n_english
if n_words < 5:
valid_english = n_english >= n_words * pct_eng_short
else:
valid_english = n_english >= n_words * pct_eng_long
return valid_english, n_words, n_english
def correct_length(words, min_words, max_words, ignore_special_tokens=True):
""" Ensure text meets threshold for containing English words
and that it's within the min and max words limits. """
if min_words is None:
min_words = 0
if max_words is None:
max_words = 99999
n_words = 0
for w in words:
if punct_word(w):
continue
if ignore_special_tokens and is_special_token(w):
continue
n_words += 1
valid = min_words <= n_words and n_words <= max_words
return valid
def punct_word(word, punctuation=string.punctuation):
return all([True if c in punctuation else False for c in word])
def load_non_english_user_set():
non_english_user_set = set(np.load('uids.npz')['data'])
return non_english_user_set
def non_english_user(userid, non_english_user_set):
neu_found = int(userid) in non_english_user_set
return neu_found
def separate_emojis_and_text(text):
emoji_chars = []
non_emoji_chars = []
for c in text:
if c in emoji.UNICODE_EMOJI:
emoji_chars.append(c)
else:
non_emoji_chars.append(c)
return ''.join(emoji_chars), ''.join(non_emoji_chars)
def extract_emojis(text, wanted_emojis):
text = remove_variation_selectors(text)
return [c for c in text if c in wanted_emojis]
def remove_variation_selectors(text):
""" Remove styling glyph variants for Unicode characters.
For instance, remove skin color from emojis.
"""
for var in VARIATION_SELECTORS:
text = text.replace(var, '')
return text
def shorten_word(word):
""" Shorten groupings of 3+ identical consecutive chars to 2, e.g. '!!!!' --> '!!'
"""
# only shorten ASCII words
try:
word.decode('ascii')
except (UnicodeDecodeError, UnicodeEncodeError, AttributeError) as e:
return word
# must have at least 3 char to be shortened
if len(word) < 3:
return word
# find groups of 3+ consecutive letters
letter_groups = [list(g) for k, g in groupby(word)]
triple_or_more = [''.join(g) for g in letter_groups if len(g) >= 3]
if len(triple_or_more) == 0:
return word
# replace letters to find the short word
short_word = word
for trip in triple_or_more:
short_word = short_word.replace(trip, trip[0]*2)
return short_word
def detect_special_tokens(word):
try:
int(word)
word = SPECIAL_TOKENS[4]
except ValueError:
if AtMentionRegex.findall(word):
word = SPECIAL_TOKENS[2]
elif urlRegex.findall(word):
word = SPECIAL_TOKENS[3]
return word
def process_word(word):
""" Shortening and converting the word to a special token if relevant.
"""
word = shorten_word(word)
word = detect_special_tokens(word)
return word
def remove_control_chars(text):
return CONTROL_CHAR_REGEX.sub('', text)
def convert_nonbreaking_space(text):
# ugly hack handling non-breaking space no matter how badly it's been encoded in the input
for r in ['\\\\xc2', '\\xc2', '\xc2', '\\\\xa0', '\\xa0', '\xa0']:
text = text.replace(r, ' ')
return text
def convert_linebreaks(text):
# ugly hack handling non-breaking space no matter how badly it's been encoded in the input
# space around to ensure proper tokenization
for r in ['\\\\n', '\\n', '\n', '\\\\r', '\\r', '\r', '<br>']:
text = text.replace(r, ' ' + SPECIAL_TOKENS[5] + ' ')
return text
|