#NEWLINE# def pipeline( urls , **kwargs ) : #NEWLINE# #INDENT# url_index = dict ( ( ( u , i ) for ( i , u ) in enumerate ( urls ) ) ) #NEWLINE# ( responses , data ) = ( ( [ None ] * len ( urls ) ) , ( [ '' ] * len ( urls ) ) ) #NEWLINE# urls = sorted ( ( re . match ( 'http://([^/]+)(.*)$' , u ) . groups ( ) for u in url_index ) ) #NEWLINE# groups = group ( urls , 0 ) #NEWLINE# for g in groups : #NEWLINE# #INDENT# pages = [ url [ 1 ] for url in g ] #NEWLINE# ( g_responses , g_data ) = raw_pipeline ( g . grouper , pages , ** kwargs ) #NEWLINE# for ( i , page ) in enumerate ( pages ) : #NEWLINE# #INDENT# url = ( 'http://%s%s' % ( g . grouper , page ) ) #NEWLINE# index = url_index [ url ] #NEWLINE# responses [ index ] = g_responses [ i ] #NEWLINE# data [ index ] = data [ i ] #NEWLINE# #UNINDENT# #UNINDENT# if ( len ( url_index ) < len ( urls ) ) : #NEWLINE# #INDENT# for ( i , res ) in enumerate ( responses ) : #NEWLINE# #INDENT# if ( res is None ) : #NEWLINE# #INDENT# index = url_index [ urls [ i ] ] #NEWLINE# responses [ i ] = responses [ index ] #NEWLINE# data [ i ] = data [ index ] #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# return ( responses , data )