#NEWLINE# def querycrawler( self , path = None , curdepth = 0 , maxdepth = 1 ) : #NEWLINE# #INDENT# self . log . debug ( ( 'Crawler is visiting %s' % path ) ) #NEWLINE# localcrawlpaths = list ( ) #NEWLINE# if ( curdepth > maxdepth ) : #NEWLINE# #INDENT# self . log . info ( ( 'maximum depth %s reached' % maxdepth ) ) #NEWLINE# return #NEWLINE# #UNINDENT# r = self . request ( path = path ) #NEWLINE# if ( r is None ) : #NEWLINE# #INDENT# return #NEWLINE# #UNINDENT# ( response , responsebody ) = r #NEWLINE# try : #NEWLINE# #INDENT# soup = BeautifulSoup ( responsebody ) #NEWLINE# #UNINDENT# except : #NEWLINE# #INDENT# self . log . warn ( 'could not parse the response body' ) #NEWLINE# return #NEWLINE# #UNINDENT# tags = soup ( 'a' ) #NEWLINE# for tag in tags : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# href = tag [ 'href' ] #NEWLINE# if ( href is not None ) : #NEWLINE# #INDENT# tmpu = urlparse ( href ) #NEWLINE# if ( ( tmpu [ 1 ] != '' ) and ( self . target != tmpu [ 1 ] ) ) : #NEWLINE# #INDENT# self . log . debug ( ( 'Ignoring link because it is not on the same site %s' % href ) ) #NEWLINE# continue #NEWLINE# #UNINDENT# if ( tmpu [ 0 ] not in [ 'http' , 'https' , '' ] ) : #NEWLINE# #INDENT# self . log . debug ( ( 'Ignoring link because it is not an http uri %s' % href ) ) #NEWLINE# continue #NEWLINE# #UNINDENT# path = tmpu [ 2 ] #NEWLINE# if ( not path . startswith ( '/' ) ) : #NEWLINE# #INDENT# path = ( '/' + path ) #NEWLINE# #UNINDENT# if ( len ( tmpu [ 4 ] ) > 0 ) : #NEWLINE# #INDENT# location = urlunparse ( ( '' , '' , tags , tmpu [ 3 ] , tmpu [ 4 ] , '' ) ) #NEWLINE# self . log . info ( ( 'Found query %s' % location ) ) #NEWLINE# return href #NEWLINE# #UNINDENT# if ( path not in self . crawlpaths ) : #NEWLINE# #INDENT# href = unquote ( path ) #NEWLINE# self . log . debug ( ( 'adding %s for crawling' % href ) ) #NEWLINE# self . crawlpaths . append ( href ) #NEWLINE# localcrawlpaths . append ( href ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# except KeyError : #NEWLINE# #INDENT# pass #NEWLINE# #UNINDENT# #UNINDENT# for nextpath in localcrawlpaths : #NEWLINE# #INDENT# r = self . querycrawler ( path = nextpath , curdepth = ( curdepth + 1 ) , maxdepth = maxdepth ) #NEWLINE# if r : #NEWLINE# #INDENT# return r