#NEWLINE# def clean_text_by_word( text , language = 'english' ) : #NEWLINE# #INDENT# ' Tokenizes a given text into words, applying filters and lemmatizing them.\n    Returns a dict of word -> syntacticUnit. ' #NEWLINE# init_textcleanner ( language ) #NEWLINE# text_without_acronyms = replace_with_separator ( text , '' , [ AB_ACRONYM_LETTERS ] ) #NEWLINE# original_words = list ( tokenize ( text_without_acronyms , to_lower = True , deacc = True ) ) #NEWLINE# filtered_words = filter_words ( original_words ) #NEWLINE# if HAS_PATTERN : #NEWLINE# #INDENT# tags = tag ( ' ' . join ( original_words ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# tags = None #NEWLINE# #UNINDENT# units = merge_syntactic_units ( original_words , filtered_words , tags ) #NEWLINE# return { unit . text : unit for unit in unit }