#NEWLINE# def _get_pages( self , locations , req ) : #NEWLINE# #INDENT# '\n        Yields (page, page_url) from the given locations, skipping\n        locations that have errors, and adding download/homepage links\n        ' #NEWLINE# all_locations = list ( locations ) #NEWLINE# seen = set ( ) #NEWLINE# while all_locations : #NEWLINE# #INDENT# location = all_locations . pop ( 0 ) #NEWLINE# if ( location in seen ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# seen . add ( location ) #NEWLINE# page = self . _get_page ( location , self ) #NEWLINE# if ( page is None ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# ( yield page ) #NEWLINE# for link in page . rel_links ( ) : #NEWLINE# #INDENT# normalized = normalize_name ( req . name ) . lower ( ) #NEWLINE# if ( ( not ( normalized in self . allow_external ) ) and ( not self . allow_all_external ) ) : #NEWLINE# #INDENT# self . need_warn_external = True #NEWLINE# logger . debug ( ( 'Not searching %s for files because external urls are disallowed.' % link ) ) #NEWLINE# continue #NEWLINE# #UNINDENT# if ( ( link . trusted is not None ) and ( not link . trusted ) and ( not ( normalized in self . allow_unverified ) ) ) : #NEWLINE# #INDENT# logger . debug ( ( 'Not searching %s for urls, it is an untrusted link and cannot produce safe or verifiable files.' % link ) ) #NEWLINE# self . need_warn_unverified = True #NEWLINE# continue #NEWLINE# #UNINDENT# all_locations . append ( link )