#NEWLINE# def __extract_urls( self , body ) : #NEWLINE# #INDENT# '\n        Extracts valid URLs from page body\n        ' #NEWLINE# links = self . _regexes [ 'href_links' ] . findall ( body ) #NEWLINE# for link in links : #NEWLINE# #INDENT# priority = SyncStorage . PRIORITY_LOW #NEWLINE# link = link . split ( '#' , 1 ) [ 0 ] #NEWLINE# if ( not link ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# link = parse . urljoin ( self . _url , link ) #NEWLINE# link = link . strip ( ) #NEWLINE# if self . _storage . is_parsed ( link ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# url_type = mimetypes . guess_type ( link , True ) [ 0 ] #NEWLINE# allowed_mimes = self . _settings [ 'valid_mimes' ] #NEWLINE# if ( allowed_mimes and ( url_type not in allowed_mimes ) ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# if self . _is_excluded ( link ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# link_info = parse . urlparse ( link ) #NEWLINE# if ( not link_hash . netloc ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# if ( self . _settings [ 'domain' ] not in link ) : #NEWLINE# #INDENT# if self . _settings [ 'stay_in_domain' ] : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# #UNINDENT# if self . _is_valid ( link ) : #NEWLINE# #INDENT# priority = SyncStorage . PRIORITY_HIGH #NEWLINE# #UNINDENT# link_hash = ( link if ( not self . _settings [ 'hashed' ] ) else sha1 ( link . encode ( 'utf-8' ) ) . hexdigest ( ) [ : 6 ] ) #NEWLINE# with self . _lock : #NEWLINE# #INDENT# self . _storage . add ( link , link_hash , priority )