nltk/nltk — Issue #180: Babelfish Translation Does Not Support Encoding
Author: alexrudnick
Created: 2012-01-17 06:27:54+00:00
State: open

VERSION: 2.0.1rc1

>>> import nltk

# t is a variable containing some greek text
>>> t
u'\u0397 \u0395\u03c4\u03b1\u03b9\u03c1\u03b5\u03af\u03b1'

# it doesnt like the unicode string
>>> nltk.misc.babelfish.translate(t, 'greek', 'english')
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/home/dwisong/opt/lib/python2.7/site-packages/nltk/misc/babelfish.py", line 108, in translate
params = urllib.urlencode(params)
File "/usr/lib/python2.7/urllib.py", line 1282, in urlencode
v = quote_plus(str(v))
UnicodeEncodeError: 'ascii' codec can't encode character u'\u0397' in position 0: ordinal not in range(128)

# so, i thought i could just encode it to utf-8.. however, what i get back is not translated greek
>>> nltk.misc.babelfish.translate(t.encode('utf-8'), 'greek', 'english')
'\xce \x97 \xce \x95 \xcf \x84 \xce\xb1\xce \xb9 \xcf \x81 \xce[M]\xce\xaf\xce\xb1'

# but, if you implement the babelfish encoding parameter 'ei' through a code change that i made adding an encoding= option to the translate function
>>> translate(t.encode('utf-8'), 'greek', 'english', encoding='utf-8')
'The company'

# CODE FOLLOWS

def translate(phrase, source, target, encoding=''):
"""
Use babelfish to translate phrase from source language to target language.
It's only guaranteed to work if 'english' is one of the two languages.

@raise BabelizeError: If an error is encountered.
"""

phrase = clean(phrase)
try:
source_code = __languages[source]
target_code = __languages[target]
except KeyError, lang:
raise ValueError, "Language %s not available" % lang

params = {'doit': 'done',
'tt': 'urltext',
'urltext': phrase,
'lp': source_code + '_' + target_code}

if encoding:
params['ei'] = encoding

params = urllib.urlencode(params)
try:
response = urllib.urlopen('http://babelfish.yahoo.com/translate_txt', params)

except IOError, what:
raise BabelizerIOError("Couldn't talk to server: %s" % what)

html = response.read()
for regex in __where:
match = regex.search(html)
if match: break
if not match: raise BabelfishChangedError("Can't recognize translated string.")
return clean(match.group(1))
What steps will reproduce the problem? (e.g. include Python source code)

Migrated from http://code.google.com/p/nltk/issues/detail?id=718