#NEWLINE# def parse_token( self , token , tags = [ WORD , POS , CHUNK , PNP , REL , ANCHOR , LEMMA ] ) : #NEWLINE# #INDENT# ' Returns the arguments for Sentence.append() from a tagged token representation.\n            The order in which token tags appear can be specified.\n            The default order is (separated by slashes): \n            - word, \n            - part-of-speech, \n            - (IOB-)chunk, \n            - (IOB-)preposition, \n            - chunk(-relation)(-role), \n            - anchor, \n            - lemma.\n            Examples:\n            The/DT/B-NP/O/NP-SBJ-1/O/the\n            cats/NNS/I-NP/O/NP-SBJ-1/O/cat\n            clawed/VBD/B-VP/O/VP-1/A1/claw\n            at/IN/B-PP/B-PNP/PP/P1/at\n            the/DT/B-NP/I-PNP/NP/P1/the\n            sofa/NN/I-NP/I-PNP/NP/P1/sofa\n            ././O/O/O/O/.\n            Returns a (word, lemma, type, chunk, role, relation, preposition, anchor, iob, custom)-tuple,\n            which can be passed to Sentence.append(): Sentence.append(*Sentence.parse_token("cats/NNS/NP"))\n            The custom value is a dictionary of (tag, value)-items of unrecognized tags in the token.\n        ' #NEWLINE# p = { #NEWLINE# #INDENT# WORD : '' , #NEWLINE# POS : None , #NEWLINE# IOB : None , #NEWLINE# CHUNK : None , #NEWLINE# PNP : None , #NEWLINE# REL : None , #NEWLINE# ROLE : None , #NEWLINE# ANCHOR : None , #NEWLINE# LEMMA : None , #NEWLINE# #UNINDENT# } #NEWLINE# custom = { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } #NEWLINE# for ( k , v ) in izip ( tags , token . split ( '/' ) ) : #NEWLINE# #INDENT# if ( SLASH0 in v ) : #NEWLINE# #INDENT# v = v . replace ( SLASH , '/' ) #NEWLINE# #UNINDENT# if ( k not in p ) : #NEWLINE# #INDENT# custom [ p ] = None #NEWLINE# #UNINDENT# if ( ( v != OUTSIDE ) or ( k == WORD ) or ( k == LEMMA ) ) : #NEWLINE# #INDENT# ( p if ( k not in custom ) else custom ) [ k ] = v #NEWLINE# #UNINDENT# #UNINDENT# ch = p [ CHUNK ] #NEWLINE# if ( ( ch is not None ) and ch . startswith ( ( 'B-' , 'I-' ) ) ) : #NEWLINE# #INDENT# ( p [ IOB ] , p [ CHUNK ] ) = ( ch [ : 1 ] , ch [ 2 : ] ) #NEWLINE# #UNINDENT# if ( p [ REL ] is not None ) : #NEWLINE# #INDENT# ( ch , p [ REL ] , p [ ROLE ] ) = self . _parse_relation ( p [ REL ] ) #NEWLINE# if ( ( ch == 'PP' ) and self . _previous and ( self . _previous [ REL ] == p [ REL ] ) and ( self . _previous [ ROLE ] == p [ ROLE ] ) ) : #NEWLINE# #INDENT# ch = 'NP' #NEWLINE# #UNINDENT# if ( ( p [ CHUNK ] is None ) and ( ch != OUTSIDE ) ) : #NEWLINE# #INDENT# p [ CHUNK ] = ch #NEWLINE# #UNINDENT# #UNINDENT# self . _previous = p #NEWLINE# return ( p [ WORD ] , p [ LEMMA ] , p [ POS ] , p [ CHUNK ] , p [ ROLE ] , p [ REL ] , p [ PNP ] , p [ ANCHOR ] , p [ IOB ] , custom )