#NEWLINE# def _find_next_token( self , pos ) : #NEWLINE# #INDENT# ' Returns a token or None if no new tokens can be found.\n        ' #NEWLINE# text = self . _text #NEWLINE# if ( pos > len ( text ) ) : #NEWLINE# #INDENT# return None #NEWLINE# #UNINDENT# match = tokenProg . search ( text , pos ) #NEWLINE# if ( not match ) : #NEWLINE# #INDENT# return None #NEWLINE# #UNINDENT# if match . group ( 1 ) : #NEWLINE# #INDENT# start = match . start ( ) #NEWLINE# end_match = endProgs [ '#' ] . search ( text , ( start + 1 ) ) #NEWLINE# end = ( end_match . start ( ) if end_match else len ( text ) ) #NEWLINE# return Token ( text , 'comment' , start , end ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( match . group ( 2 ) is not None ) : #NEWLINE# #INDENT# start = match . start ( ) #NEWLINE# string_style = match . group ( 3 ) #NEWLINE# end = endProgs [ string_style ] . search ( text , ( match . end ( ) - 1 ) ) . end ( ) #NEWLINE# return Token ( text , 'string' , start , end ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# identifier = match . group ( 4 ) #NEWLINE# tokenArgs = ( match . start ( ) , match . end ( ) ) #NEWLINE# if ( identifier in KEYWORDS ) : #NEWLINE# #INDENT# return Token ( text , 'keyword' , * tokenArgs ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( identifier [ 0 ] in '0123456789' ) : #NEWLINE# #INDENT# return Token ( text , 'number' , * tokenArgs ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# return Token ( text , 'identifier' , * tokenArgs )