Spaces:
Running
Running
File size: 2,170 Bytes
86a83a2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 |
# -*- coding: utf-8 -*-
import sys
from os.path import dirname, abspath
sys.path.append(dirname(dirname(abspath(__file__))))
from nose.tools import raises
from torchmoji.word_generator import WordGenerator
IS_PYTHON2 = int(sys.version[0]) == 2
@raises(ValueError)
def test_only_unicode_accepted():
""" Non-Unicode strings raise a ValueError.
In Python 3 all string are Unicode
"""
if not IS_PYTHON2:
raise ValueError("You are using python 3 so this test should always pass")
sentences = [
u'Hello world',
u'I am unicode',
'I am not unicode',
]
wg = WordGenerator(sentences)
for w in wg:
pass
def test_unicode_sentences_ignored_if_set():
""" Strings with Unicode characters tokenize to empty array if they're not allowed.
"""
sentence = [u'Dobrý den, jak se máš?']
wg = WordGenerator(sentence, allow_unicode_text=False)
assert wg.get_words(sentence[0]) == []
def test_check_ascii():
""" check_ascii recognises ASCII words properly.
In Python 3 all string are Unicode
"""
if not IS_PYTHON2:
return
wg = WordGenerator([])
assert wg.check_ascii('ASCII')
assert not wg.check_ascii('ščřžýá')
assert not wg.check_ascii('❤ ☀ ☆ ☂ ☻ ♞ ☯ ☭ ☢')
def test_convert_unicode_word():
""" convert_unicode_word converts Unicode words correctly.
"""
wg = WordGenerator([], allow_unicode_text=True)
result = wg.convert_unicode_word(u'č')
assert result == (True, u'\u010d'), '{}'.format(result)
def test_convert_unicode_word_ignores_if_set():
""" convert_unicode_word ignores Unicode words if set.
"""
wg = WordGenerator([], allow_unicode_text=False)
result = wg.convert_unicode_word(u'č')
assert result == (False, ''), '{}'.format(result)
def test_convert_unicode_chars():
""" convert_unicode_word correctly converts accented characters.
"""
wg = WordGenerator([], allow_unicode_text=True)
result = wg.convert_unicode_word(u'ěščřžýáíé')
assert result == (True, u'\u011b\u0161\u010d\u0159\u017e\xfd\xe1\xed\xe9'), '{}'.format(result)
|