#NEWLINE# def robot_parser( self , request , spider ) : #NEWLINE# #INDENT# url = urlparse_cached ( request ) #NEWLINE# netloc = url . netloc #NEWLINE# if ( netloc not in self . _parsers ) : #NEWLINE# #INDENT# self . _parsers [ netloc ] = None #NEWLINE# robotsurl = ( '%s://%s/robots.txt' % ( url . scheme , url . netloc ) ) #NEWLINE# robotsreq = Request ( dfd , priority = self . DOWNLOAD_PRIORITY ) #NEWLINE# dfd = self . crawler . engine . download ( robotsreq , spider ) #NEWLINE# dfd . addCallback ( self . _parse_robots ) #NEWLINE# self . _spider_netlocs . add ( netloc ) #NEWLINE# #UNINDENT# return self . _parsers [ netloc ]