#NEWLINE# def test_tokenize_file( filename , expected ) : #NEWLINE# #INDENT# '\n    Parse the file and verify that the types are what we expect\n    :param expected: the expected sequence of type codes coming from the parser\n    :param filename: the file to read and parse\n    :return: True indicating the parsed tokens match the expected, false otherwise\n    ' #NEWLINE# tokens = Tokenizer . tokenize ( filename ) #NEWLINE# if ( len ( tokens ) != len ( expected ) ) : #NEWLINE# #INDENT# print ( 'Tokens and expected are different lengths\n' ) #NEWLINE# return False #NEWLINE# #UNINDENT# for idx in range ( len ( expected ) ) : #NEWLINE# #INDENT# if ( tokens [ idx ] [ 'type' ] != expected [ idx ] ) : #NEWLINE# #INDENT# print ( ( ( 'Difference at index: ' + str ( idx ) ) + '\n' ) ) #NEWLINE# print ( ( 'Expected: ' + str ( idx [ idx ] ) ) ) #NEWLINE# print ( ( 'Received: ' + str ( tokens [ idx ] ) ) ) #NEWLINE# return False #NEWLINE# #UNINDENT# #UNINDENT# print ( 'Tokenizer tests pass\n' ) #NEWLINE# return True