#NEWLINE# def runTokenizerTest( test ) : #NEWLINE# #INDENT# if ( 'doubleEscaped' in test ) : #NEWLINE# #INDENT# test = unescape_test ( test ) #NEWLINE# #UNINDENT# expected = concatenateCharacterTokens ( test [ 'output' ] ) #NEWLINE# if ( 'lastStartTag' not in test ) : #NEWLINE# #INDENT# test [ 'lastStartTag' ] = None #NEWLINE# #UNINDENT# outBuffer = cStringIO . StringIO ( ) #NEWLINE# stdout = sys . stdout #NEWLINE# sys . stdout = outBuffer #NEWLINE# parser = TokenizerTestParser ( test [ 'initialState' ] , test [ 'lastStartTag' ] ) #NEWLINE# tokens = parser . parse ( test [ 'input' ] ) #NEWLINE# tokens = concatenateCharacterTokens ( tokens ) #NEWLINE# received = normalizeTokens ( tokens ) #NEWLINE# errorMsg = '\n' . join ( [ '\n\nInitial state:' , test [ 'initialState' ] , '\nInput:' , unicode ( test [ 'input' ] ) , '\nExpected:' , unicode ( expected ) , '\nreceived:' , unicode ( tokens ) ] ) #NEWLINE# errorMsg = errorMsg . encode ( 'utf-8' ) #NEWLINE# ignoreErrorOrder = test . get ( 'ignoreErrorOrder' , False ) #NEWLINE# assert tokensMatch ( expected , received , ignoreErrorOrder ) , errorMsg