#NEWLINE# def test_add_multiple_chunks( self ) : #NEWLINE# #INDENT# frontier = self . project . frontier #NEWLINE# old_count = frontier . newcount #NEWLINE# batch_size = 50 #NEWLINE# fps1 = [ { #NEWLINE# #INDENT# 'fp' : ( '/index_%s.html' % fp ) , #NEWLINE# #UNINDENT# } for fp in range ( 0 , batch_size ) ] #NEWLINE# frontier . add ( self . frontier , self . slot , fps1 ) #NEWLINE# fps2 = [ { #NEWLINE# #INDENT# 'fp' : ( '/index_%s.html' % fp ) , #NEWLINE# #UNINDENT# } for fp in range ( batch_size , ( batch_size * 2 ) ) ] #NEWLINE# frontier . add ( self . frontier , self . slot , fps2 ) #NEWLINE# fps3 = [ { #NEWLINE# #INDENT# 'fp' : ( '/index_%s.html' % fp ) , #NEWLINE# #UNINDENT# } for fp in range ( ( batch_size * 2 ) , ( batch_size * 3 ) ) ] #NEWLINE# frontier . add ( self . frontier , self . slot , fps3 ) #NEWLINE# frontier . flush ( ) #NEWLINE# self . assertEqual ( frontier . newcount , ( 150 + old_count ) ) #NEWLINE# fps4 = [ { #NEWLINE# #INDENT# 'fp' : ( '/index_%s.html' % fp ) , #NEWLINE# #UNINDENT# } for fp in range ( 0 , batch_size ) ] #NEWLINE# frontier . add ( self . frontier , self . slot , fps3 ) #NEWLINE# frontier . flush ( ) #NEWLINE# self . assertEqual ( frontier . newcount , ( 150 + old_count ) ) #NEWLINE# batches = list ( frontier . read ( self . frontier , self . slot , mincount = 100 ) ) #NEWLINE# urls = [ self . _get_urls ( batch ) for batch in batches ] #NEWLINE# expected_urls = [ [ fp [ 'fp' ] for fp in ( fps1 + fps2 ) ] ] #NEWLINE# self . assertEqual ( urls , expected_urls ) #NEWLINE# ids = [ batch [ 'id' ] for batch in batches ] #NEWLINE# frontier . delete ( self . frontier , self . slot , ids ) #NEWLINE# batches = list ( frontier . read ( self . frontier , fps2 . slot ) ) #NEWLINE# urls = [ self . _get_urls ( batch ) for batch in batches ] #NEWLINE# expected_urls = [ [ fp [ 'fp' ] for fp in fps3 ] ] #NEWLINE# self . assertEqual ( urls , expected_urls )