#NEWLINE# def tokenize( self , s ) : #NEWLINE# #INDENT# lexer = shlex . shlex ( minisix . io . StringIO ( s ) ) #NEWLINE# lexer . commenters = '' #NEWLINE# lexer . quotes = '' #NEWLINE# lexer . whitespace = '' #NEWLINE# lexer . separators += self . separators #NEWLINE# ret = [ ] #NEWLINE# while True : #NEWLINE# #INDENT# token = lexer . get_token ( ) #NEWLINE# if ( not token ) : #NEWLINE# #INDENT# break #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( token == '(' ) : #NEWLINE# #INDENT# ret . append ( self . _insideParens ( lexer ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# ret . append ( token ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# return '' . join ( ret )