#NEWLINE# def __fetch_pages( self ) : #NEWLINE# #INDENT# '\n        Main fetching loop\n        ' #NEWLINE# urls = self . _storage . pop ( self . _settings [ 'threads' ] ) #NEWLINE# lock = ( Lock ( ) if self . _settings [ 'with_lock' ] else DummyLock ( ) ) #NEWLINE# while urls : #NEWLINE# #INDENT# active_threads = [ ] #NEWLINE# for url in urls : #NEWLINE# #INDENT# fetcher = Fetcher ( ) #NEWLINE# fetcher . setup ( { #NEWLINE# #INDENT# 'url' : url , #NEWLINE# 'lock' : lock , #NEWLINE# 'parent' : self , #NEWLINE# #UNINDENT# } ) #NEWLINE# fetcher . start ( ) #NEWLINE# active_threads . append ( fetcher ) #NEWLINE# #UNINDENT# for fetcher in active_threads : #NEWLINE# #INDENT# fetcher . join ( ) #NEWLINE# #UNINDENT# urls = self . _storage . pop ( self . _settings [ 'threads' ] )