#NEWLINE# def _parse_tokens( chunk , format = [ WORD , POS , CHUNK , PNP , REL , ANCHOR , LEMMA ] ) : #NEWLINE# #INDENT# ' Parses tokens from <word> elements in the given XML <chunk> element.\n        Returns a flat list of tokens, in which each token is [WORD, POS, CHUNK, PNP, RELATION, ANCHOR, LEMMA].\n        If a <chunk type="PNP"> is encountered, traverses all of the chunks in the PNP.\n    ' #NEWLINE# tokens = [ ] #NEWLINE# if ( not ( ( chunk . tag == XML_CHUNK ) or ( chunk . tag == XML_CHINK ) ) ) : #NEWLINE# #INDENT# return [ ] #NEWLINE# #UNINDENT# type = chunk . get ( XML_TYPE , 'O' ) #NEWLINE# if ( type == 'PNP' ) : #NEWLINE# #INDENT# for ch in chunk : #NEWLINE# #INDENT# tokens . extend ( _parse_tokens ( ch , format ) ) #NEWLINE# #UNINDENT# if ( PNP in format ) : #NEWLINE# #INDENT# i = format . index ( PNP ) #NEWLINE# for ( j , token ) in enumerate ( tokens ) : #NEWLINE# #INDENT# token [ i ] = ( ( ( ( j == 0 ) and 'B-' ) or 'I-' ) + 'PNP' ) #NEWLINE# #UNINDENT# #UNINDENT# a = chunk . get ( XML_OF ) . split ( _UID_SEPARATOR ) [ ( - 1 ) ] #NEWLINE# if a : #NEWLINE# #INDENT# _attachments . setdefault ( a , [ ] ) #NEWLINE# _attachments [ a ] . append ( tokens ) #NEWLINE# #UNINDENT# return tokens #NEWLINE# #UNINDENT# relation = _parse_relation ( chunk , format ) #NEWLINE# for word in filter ( ( lambda n : ( n . tag == XML_WORD ) ) , chunk ) : #NEWLINE# #INDENT# tokens . append ( _parse_token ( word , chunk = type , relation = relation , format = format ) ) #NEWLINE# #UNINDENT# if ( CHUNK in format ) : #NEWLINE# #INDENT# i = format . index ( CHUNK ) #NEWLINE# for ( j , token ) in enumerate ( tokens ) : #NEWLINE# #INDENT# token [ i ] = ( ( ( token [ i ] != 'O' ) and ( ( ( ( j == 0 ) and 'B-' ) or 'I-' ) + token [ i ] ) ) or 'O' ) #NEWLINE# #UNINDENT# #UNINDENT# a = chunk . get ( XML_ANCHOR , '' ) . split ( _UID_SEPARATOR ) [ ( - 1 ) ] #NEWLINE# if a : #NEWLINE# #INDENT# _anchors [ a ] = tokens #NEWLINE# #UNINDENT# return tokens