#NEWLINE# def merge_all_files_into_pytables( file_dir , file_out ) : #NEWLINE# #INDENT# '\n    process each file into pytables\n    ' #NEWLINE# start = None #NEWLINE# start = datetime . datetime . now ( ) #NEWLINE# out_h5 = tables . openFile ( file_out , mode = 'w' , title = 'bars' , filters = tables . Filters ( complevel = 9 , complib = 'zlib' ) ) #NEWLINE# table = None #NEWLINE# for file_in in glob . glob ( ( file_dir + '/*.gz' ) ) : #NEWLINE# #INDENT# gzip_file = gzip . open ( file_in ) #NEWLINE# expected_header = [ 'dt' , 'sid' , 'open' , 'high' , 'low' , 'close' , 'volume' ] #NEWLINE# csv_reader = csv . DictReader ( gzip_file ) #NEWLINE# header = csv_reader . fieldnames #NEWLINE# if ( header != expected_header ) : #NEWLINE# #INDENT# logging . warn ( ( 'expected header %s\n' % expected_header ) ) #NEWLINE# logging . warn ( ( 'header_found %s' % header ) ) #NEWLINE# return #NEWLINE# #UNINDENT# for ( current_date , rows ) in parse_csv ( csv_reader ) : #NEWLINE# #INDENT# table = out_h5 . createTable ( '/TD' , ( 'date_' + current_date ) , OHLCTableDescription , expectedrows = len ( rows ) , createparents = True ) #NEWLINE# table . append ( rows ) #NEWLINE# table . flush ( ) #NEWLINE# #UNINDENT# if ( diff is not None ) : #NEWLINE# #INDENT# table . flush ( ) #NEWLINE# #UNINDENT# #UNINDENT# end = datetime . datetime . now ( ) #NEWLINE# diff = ( end - start ) . seconds #NEWLINE# logging . debug ( ( 'finished  it took %d.' % diff ) )