#NEWLINE# def robot_parser( self , request , spider ) : #NEWLINE# #INDENT# url = urlparse_cached ( request ) #NEWLINE# netloc = url . netloc #NEWLINE# if ( netloc not in self . _parsers ) : #NEWLINE# #INDENT# self . _parsers [ netloc ] = Deferred ( ) #NEWLINE# robotsurl = ( '%s://%s/robots.txt' % ( url . scheme , url . netloc ) ) #NEWLINE# robotsreq = Request ( robotsurl , priority = self . DOWNLOAD_PRIORITY , meta = { #NEWLINE# #INDENT# 'dont_obey_robotstxt' : True , #NEWLINE# #UNINDENT# } ) #NEWLINE# dfd = self . crawler . engine . download ( robotsreq , spider ) #NEWLINE# dfd . addCallback ( self . _parse_robots , netloc ) #NEWLINE# dfd . addErrback ( self . _logerror , robotsreq , spider ) #NEWLINE# dfd . addErrback ( self . _robots_error , netloc ) #NEWLINE# #UNINDENT# if isinstance ( d . _parsers [ netloc ] , Deferred ) : #NEWLINE# #INDENT# d = Deferred ( ) #NEWLINE# def cb( result ) : #NEWLINE# #INDENT# d . callback ( result ) #NEWLINE# return result #NEWLINE# #UNINDENT# self . _parsers [ netloc ] . addCallback ( cb ) #NEWLINE# return d #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# return self . _parsers [ netloc ]