#NEWLINE# @ click . command ( 'tokens2pos' ) #NEWLINE# @ click . argument ( 'tokens' , type = click . File ( 'r' ) , default = click . open_file ( '-' ) ) #NEWLINE# @ click . option ( '--sep' , default = ' ' , help = 'Separator between words in the output.' , show_default = True ) #NEWLINE# def tokens2pos( sep , tokens ) : #NEWLINE# #INDENT# 'Tokenize words into their parts of speech. Each item\n    is the original word with its role as the second part\n    of the item. Punctuation is considered as a separate token.' #NEWLINE# content = read_tokens ( tokens ) #NEWLINE# nltk . data . path . append ( data_item ( ) ) #NEWLINE# tags = nltk . pos_tag ( content ) #NEWLINE# [ output ( '{},{}' . format ( t [ 0 ] , t [ 1 ] ) ) for t in tags ]