#NEWLINE# def build_spider_registry( config ) : #NEWLINE# #INDENT# SPIDER_REGISTRY . clear ( ) #NEWLINE# opt_modules = [ ] #NEWLINE# opt_modules = config [ 'global' ] . get ( 'spider_modules' , [ ] ) #NEWLINE# for path in opt_modules : #NEWLINE# #INDENT# if ( ':' in path ) : #NEWLINE# #INDENT# ( path , cls_name ) = path . split ( ':' ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# cls_name = None #NEWLINE# #UNINDENT# try : #NEWLINE# #INDENT# mod = __import__ ( path , None , None , [ 'foo' ] ) #NEWLINE# #UNINDENT# except ImportError as ex : #NEWLINE# #INDENT# if ( path not in six . text_type ( ex ) ) : #NEWLINE# #INDENT# logging . error ( '' , exc_info = ex ) #NEWLINE# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# for key in dir ( mod ) : #NEWLINE# #INDENT# if ( key == 'Spider' ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# if ( ( cls_name is None ) or ( key == cls_name ) ) : #NEWLINE# #INDENT# val = getattr ( mod , key ) #NEWLINE# if ( isinstance ( val , type ) and issubclass ( val , Spider ) ) : #NEWLINE# #INDENT# if val . Meta . abstract : #NEWLINE# #INDENT# pass #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# spider_name = val . get_spider_name ( ) #NEWLINE# logger . debug ( ( 'Module `%s`, found spider `%s` with name `%s`' % ( path , val . __name__ , spider_name ) ) ) #NEWLINE# if ( spider_name in SPIDER_REGISTRY ) : #NEWLINE# #INDENT# mod = SPIDER_REGISTRY [ spider_name ] . __module__ #NEWLINE# raise SpiderInternalError ( ( 'There are two different spiders with the same name "%s". Modules: %s and %s' % ( spider_name , mod , val . __module__ ) ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# SPIDER_REGISTRY [ spider_name ] = val #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# return SPIDER_REGISTRY