#NEWLINE# def _feature_detector( self , tokens , index , history ) : #NEWLINE# #INDENT# word = tokens [ index ] [ 0 ] #NEWLINE# pos = simplify_pos ( tokens [ index ] [ 1 ] ) #NEWLINE# if ( index == 0 ) : #NEWLINE# #INDENT# prevword = prevprevword = None #NEWLINE# prevpos = prevprevpos = None #NEWLINE# prevshape = prevtag = prevprevtag = None #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( index == 1 ) : #NEWLINE# #INDENT# prevword = tokens [ ( index - 1 ) ] [ 0 ] . lower ( ) #NEWLINE# prevprevword = None #NEWLINE# prevpos = simplify_pos ( tokens [ ( index - 1 ) ] [ 1 ] ) #NEWLINE# prevprevpos = None #NEWLINE# prevtag = history [ ( index - 1 ) ] [ 0 ] #NEWLINE# prevshape = prevprevtag = None #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# prevword = tokens [ ( index - 1 ) ] [ 0 ] . lower ( ) #NEWLINE# prevprevword = tokens [ ( index - 2 ) ] [ 0 ] . lower ( ) #NEWLINE# prevpos = simplify_pos ( tokens [ ( index - 1 ) ] [ 1 ] ) #NEWLINE# prevprevpos = simplify_pos ( tokens [ ( index - 2 ) ] [ 1 ] ) #NEWLINE# prevtag = history [ ( index - 1 ) ] #NEWLINE# prevprevtag = history [ ( index - 2 ) ] #NEWLINE# prevshape = shape ( prevword ) #NEWLINE# #UNINDENT# #UNINDENT# if ( index == ( len ( tokens ) - 1 ) ) : #NEWLINE# #INDENT# nextword = nextnextword = None #NEWLINE# nextpos = nextnextpos = None #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( index == ( len ( tokens ) - 2 ) ) : #NEWLINE# #INDENT# nextword = tokens [ ( index + 1 ) ] [ 0 ] . lower ( ) #NEWLINE# nextpos = tokens [ ( index + 1 ) ] [ 1 ] . lower ( ) #NEWLINE# nextnextword = None #NEWLINE# nextnextpos = None #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# nextword = tokens [ ( index + 1 ) ] [ 0 ] . lower ( ) #NEWLINE# nextpos = tokens [ ( index + 1 ) ] [ 1 ] . lower ( ) #NEWLINE# nextnextword = tokens [ ( index + 2 ) ] [ 0 ] . lower ( ) #NEWLINE# nextnextpos = tokens [ ( index + 2 ) ] [ 1 ] . lower ( ) #NEWLINE# #UNINDENT# #UNINDENT# features = { #NEWLINE# #INDENT# 'bias' : True , #NEWLINE# 'shape' : shape ( word ) , #NEWLINE# 'wordlen' : len ( word ) , #NEWLINE# 'prefix3' : word [ : 3 ] . lower ( ) , #NEWLINE# 'suffix3' : word [ ( - 3 ) : ] . lower ( ) , #NEWLINE# 'pos' : pos , #NEWLINE# 'word' : word , #NEWLINE# 'en-wordlist' : ( word in self . _english_wordlist ( ) ) , #NEWLINE# 'prevtag' : prevtag , #NEWLINE# 'prevpos' : prevpos , #NEWLINE# 'nextpos' : nextpos , #NEWLINE# 'prevword' : prevword , #NEWLINE# 'nextword' : nextword , #NEWLINE# 'word+nextpos' : ( '%s+%s' % ( word . lower ( ) , nextpos ) ) , #NEWLINE# 'pos+prevtag' : ( '%s+%s' % ( pos , prevtag ) ) , #NEWLINE# 'shape+prevtag' : ( '%s+%s' % ( prevshape , prevtag ) ) , #NEWLINE# #UNINDENT# } #NEWLINE# return features