#NEWLINE# def _should_queue( self , link , referrer , rel ) : #NEWLINE# #INDENT# '\n        Determine whether a link URL from a referring page and with a\n        particular "rel" attribute should be queued for scraping.\n        ' #NEWLINE# ( scheme , netloc , path , _ , _ , _ ) = urlparse ( link ) #NEWLINE# if path . endswith ( ( ( self . source_extensions + self . binary_extensions ) + self . excluded_extensions ) ) : #NEWLINE# #INDENT# result = False #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( self . skip_externals and ( not link . startswith ( self . base_url ) ) ) : #NEWLINE# #INDENT# result = False #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( not referrer . startswith ( self . base_url ) ) : #NEWLINE# #INDENT# result = False #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( rel not in ( 'homepage' , 'download' ) ) : #NEWLINE# #INDENT# result = False #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( scheme not in ( 'http' , 'https' , 'ftp' ) ) : #NEWLINE# #INDENT# result = False #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if self . _is_platform_dependent ( link ) : #NEWLINE# #INDENT# result = False #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# host = netloc . split ( ':' , 1 ) [ 0 ] #NEWLINE# if ( host . lower ( ) == 'localhost' ) : #NEWLINE# #INDENT# result = False #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# result = True #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# logger . debug ( 'should_queue: %s (%s) from %s -> %s' , link , rel , referrer , result ) #NEWLINE# return result