#NEWLINE# def scanTokens( msg , whitespace ) : #NEWLINE# #INDENT# curVal = '' #NEWLINE# curType = None #NEWLINE# scan = [ ] #NEWLINE# for m in msg : #NEWLINE# #INDENT# if ( m in whitespace ) : #NEWLINE# #INDENT# if ( curType is None ) : #NEWLINE# #INDENT# curType = WS #NEWLINE# curVal = m #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( curType is TOK ) : #NEWLINE# #INDENT# scan . append ( ( TOK , curVal ) ) #NEWLINE# curType = WS #NEWLINE# curVal = m #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# curVal += m #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# if ( curType is None ) : #NEWLINE# #INDENT# curType = TOK #NEWLINE# curVal = m #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( curType is WS ) : #NEWLINE# #INDENT# scan . append ( ( WS , curVal ) ) #NEWLINE# curType = TOK #NEWLINE# curVal = m #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# curVal += m #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# if ( curType is not None ) : #NEWLINE# #INDENT# if ( curType is TOK ) : #NEWLINE# #INDENT# scan . append ( ( TOK , curVal ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# scan . append ( ( WS , curVal ) ) #NEWLINE# #UNINDENT# #UNINDENT# return scan