#NEWLINE# def u_clean( s ) : #NEWLINE# #INDENT# 'A strange attempt at cleaning up unicode' #NEWLINE# uni = '' #NEWLINE# try : #NEWLINE# #INDENT# uni = str ( s ) . decode ( 'iso-8859-1' ) #NEWLINE# #UNINDENT# except UnicodeDecodeError : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# uni = str ( s ) . decode ( 'utf-8' ) #NEWLINE# #UNINDENT# except UnicodeDecodeError : #NEWLINE# #INDENT# if ( s and ( type ( s ) in ( str , unicode ) ) ) : #NEWLINE# #INDENT# for c in s : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# uni += unicodedata . normalize ( 'NFKC' , unicode ( c ) ) #NEWLINE# #UNINDENT# except UnicodeDecodeError : #NEWLINE# #INDENT# uni += '-' #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# return uni . encode ( 'ascii' , 'xmlcharrefreplace' )